Home
last modified time | relevance | path

Searched full:precision (Results 1 – 25 of 168) sorted by relevance

1234567

/arkcompiler/runtime_core/static_core/plugins/ets/tests/interop_js/tests/declgen_ets2ts/fields/
Dfields.ts27 const PRECISION = 3; constant
46 assertEq(Fields.staticFloat_.toFixed(PRECISION), (ONE + ONE_TENTH).toFixed(PRECISION));
47 assertEq(fields.number_.toFixed(PRECISION), (TWO + TWO_TENTH).toFixed(PRECISION));
48 assertEq(fields.Number_.toFixed(PRECISION), (THREE + THREE_TENTH).toFixed(PRECISION));
49 assertEq(fields.double_.toFixed(PRECISION), (FOUR + FOUR_TENTH).toFixed(PRECISION));
50 assertEq(fields.Double_.toFixed(PRECISION), (FIVE + FIVE_TENTH).toFixed(PRECISION));
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets_sdk/arkts/@arkts/math/Decimal/
DsetTests.ets25 suite.addTest("Decimal set with precision", testWithPrecision);
40 arktest.assertEQ(Decimal.precision, expected);
79 …Decimal.set({ precision: 20, rounding: 4, toExpNeg: -7, toExpPos: 21, minE: -9e15, maxE: 9e15, cry…
80 arktest.assertEQ(Decimal.precision, 20);
89 …Decimal.set({ precision: 40, rounding : 4, toExpNeg: -1000, toExpPos: 1000, minE: -1e9, maxE: 1e9,…
90 arktest.assertEQ(Decimal.precision, 40);
106 tPrecision(1, { precision: 1 });
107 tPrecision(20, { precision: 20 });
108 tPrecision(300000, { precision: 300000 });
109 tPrecision(4e+8, { precision: 4e8 });
[all …]
DisFiniteTests.ets34 … Decimal.set({ precision: 20, rounding: 4, toExpNeg: -7, toExpPos: 21, minE: -9e15, maxE: 9e15 });
44 … Decimal.set({ precision: 20, rounding: 4, toExpNeg: -7, toExpPos: 21, minE: -9e15, maxE: 9e15 });
54 … Decimal.set({ precision: 20, rounding: 4, toExpNeg: -7, toExpPos: 21, minE: -9e15, maxE: 9e15 });
64 … Decimal.set({ precision: 20, rounding: 4, toExpNeg: -7, toExpPos: 21, minE: -9e15, maxE: 9e15 });
74 … Decimal.set({ precision: 20, rounding: 4, toExpNeg: -7, toExpPos: 21, minE: -9e15, maxE: 9e15 });
84 … Decimal.set({ precision: 20, rounding: 4, toExpNeg: -7, toExpPos: 21, minE: -9e15, maxE: 9e15 });
93 … Decimal.set({ precision: 20, rounding: 4, toExpNeg: -7, toExpPos: 21, minE: -9e15, maxE: 9e15 });
103 … Decimal.set({ precision: 20, rounding: 4, toExpNeg: -7, toExpPos: 21, minE: -9e15, maxE: 9e15 });
113 … Decimal.set({ precision: 20, rounding: 4, toExpNeg: -7, toExpPos: 21, minE: -9e15, maxE: 9e15 });
123 … Decimal.set({ precision: 20, rounding: 4, toExpNeg: -7, toExpPos: 21, minE: -9e15, maxE: 9e15 });
/arkcompiler/runtime_core/static_core/plugins/ets/tests/interop_js/tests/declgen_ets2ts/functions/
Dfunctions.ts21 const PRECISION = 3; constant
38 …umDouble(THREE, TWENTY_TWO_HUNDREDTH).toFixed(PRECISION), (THREE + TWENTY_TWO_HUNDREDTH).toFixed(P…
39 …edDouble(THREE, TWENTY_TWO_HUNDREDTH).toFixed(PRECISION), (THREE + TWENTY_TWO_HUNDREDTH).toFixed(P…
/arkcompiler/runtime_core/static_core/plugins/ets/tests/interop_js/tests/declgen_ets2ts/methods/
Dmethods.ts23 const PRECISION = 3; constant
43 …umDouble(THREE, TWENTY_TWO_HUNDREDTH).toFixed(PRECISION), (THREE + TWENTY_TWO_HUNDREDTH).toFixed(P…
55 …rtEq(o.SumIntVariadic(ONE, TWO, THREE, FOUR, FIVE).toFixed(PRECISION), FIFTEEN.toFixed(PRECISION));
/arkcompiler/runtime_core/static_core/plugins/ets/tests/interop_js/tests/declgen_ets2ts/generics/
Dgenerics.ts21 const PRECISION = 3; constant
51 assertEq(tNumber.identity(FIVE).toFixed(PRECISION), FIVE.toFixed(PRECISION));
53 …dentity(THREE + TWENTY_TWO_HUNDREDTH).toFixed(PRECISION), (THREE + TWENTY_TWO_HUNDREDTH).toFixed(P…
/arkcompiler/runtime_core/static_core/plugins/ets/tests/interop_js/tests/declgen_ets2ts/imports/
Dimports.ts19 const PRECISION = 3; constant
39 assertEq(o.v1.toFixed(PRECISION), ONE.toFixed(PRECISION));
/arkcompiler/runtime_core/static_core/plugins/ets/tests/stdlib-templates/utils/
Dtest_escompat_dataview.j245 …etter_base}}(0, true))).toPrecision({{.item.precision}}) != (new Number(value)).toPrecision({{.ite…
53 … ((new Number(value)).toPrecision({{.item.precision}}) != (new Number(reversed)).toPrecision({{.it…
74 const goldString = new Number(value).toPrecision({{.item.precision}})
78 const fromBufferString = fromBuffer.toPrecision({{.item.precision}})
/arkcompiler/runtime_core/static_core/libpandabase/utils/
Dtype_converter.h37 * @param precision_ - precision for output
46 * @brief Set new @param precision for output
47 * @param precision
49 void SetPrecision(size_t precision);
/arkcompiler/runtime_core/libpandabase/utils/
Dtype_converter.h37 * @param precision_ - precision for output
46 * \brief Set new \param precision for output
47 * @param precision
49 void SetPrecision(size_t precision);
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets_func_tests/std/math/
DRoundTest.ets44 …"round(3.5000000000000000000000000001)") //Exceeds double storage precision, actual storage …
45 …, "round(-3.5000000000000000000000000001)") //Exceeds double storage precision, actual storage …
48 …"round(5.4999999999999999999999999999)") //Exceeds double storage precision, actual storage …
49 …, "round(-5.4999999999999999999999999999)") //Exceeds double storage precision, actual storage …
/arkcompiler/ets_runtime/ecmascript/
Djs_number_format.h182 icu::number::Precision precision = in SetICUFormatterDigitOptions() local
183 icu::number::Precision::minMaxFraction(minimumFractionDigits, maximumFractionDigits); in SetICUFormatterDigitOptions()
188 precision = in SetICUFormatterDigitOptions()
189 …icu::number::Precision::minMaxSignificantDigits(minimumSignificantDigits, maximumSignificantDigits… in SetICUFormatterDigitOptions()
191 return icuNumberformatter.precision(precision); in SetICUFormatterDigitOptions()
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets-templates_deprecated/05.unboxing_conversion/call_lmbd/
Dcall-lmbd_2.params.yaml106 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
107 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
128 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
129 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
130 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
131 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
155 - { expr: new Float(Int.MIN_VALUE as float), val: -2.147483648E9 } # loss of precision
156 - { expr: new Float(Int.MAX_VALUE as float), val: 2.147483648E9 } # loss of precision
157 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372036854776E18 } # loss of precision
158 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372036854776E18 } # loss of precision
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets-templates_deprecated/05.unboxing_conversion/call_meth/
Dcall-meth.params.yaml133 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
134 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
168 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
169 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
170 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
171 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
195 … - { expr: new Float(Int.MIN_VALUE as float), val: -2.14748365E9 as float } # loss of precision
196 … - { expr: new Float(Int.MAX_VALUE as float), val: 2.14748365E9 as float } # loss of precision
197 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372E18 as float } # loss of precision
198 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372E18 as float } # loss of precision
[all …]
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets-templates_deprecated/05.unboxing_conversion/assn_var/
Dassn-var.params.yaml133 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
134 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
168 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
169 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
170 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
171 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
195 … - { expr: new Float(Int.MIN_VALUE as float), val: -2.14748365E9 as float } # loss of precision
196 … - { expr: new Float(Int.MAX_VALUE as float), val: 2.14748365E9 as float } # loss of precision
197 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372E18 as float } # loss of precision
198 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372E18 as float } # loss of precision
[all …]
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets-templates_deprecated/05.unboxing_conversion/comp_obj/
Dcomp-intf.params.yaml133 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
134 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
168 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
169 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
170 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
171 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
195 … - { expr: new Float(Int.MIN_VALUE as float), val: -2.14748365E9 as float } # loss of precision
196 … - { expr: new Float(Int.MAX_VALUE as float), val: 2.14748365E9 as float } # loss of precision
197 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372E18 as float } # loss of precision
198 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372E18 as float } # loss of precision
[all …]
Dcomp-clss.params.yaml133 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
134 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
168 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
169 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
170 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
171 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
195 … - { expr: new Float(Int.MIN_VALUE as float), val: -2.14748365E9 as float } # loss of precision
196 … - { expr: new Float(Int.MAX_VALUE as float), val: 2.14748365E9 as float } # loss of precision
197 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372E18 as float } # loss of precision
198 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372E18 as float } # loss of precision
[all …]
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets-templates_deprecated/05.unboxing_conversion/call_cons/
Dcall-cons.params.yaml133 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
134 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
168 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
169 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
170 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
171 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
195 … - { expr: new Float(Int.MIN_VALUE as float), val: -2.14748365E9 as float } # loss of precision
196 … - { expr: new Float(Int.MAX_VALUE as float), val: 2.14748365E9 as float } # loss of precision
197 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372E18 as float } # loss of precision
198 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372E18 as float } # loss of precision
[all …]
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets-templates_deprecated/05.unboxing_conversion/comp_arr/
Dcomp-arr.params.yaml133 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
134 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
168 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
169 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
170 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
171 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
195 … - { expr: new Float(Int.MIN_VALUE as float), val: -2.14748365E9 as float } # loss of precision
196 … - { expr: new Float(Int.MAX_VALUE as float), val: 2.14748365E9 as float } # loss of precision
197 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372E18 as float } # loss of precision
198 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372E18 as float } # loss of precision
[all …]
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets-templates_deprecated/05.unboxing_conversion/decl_field/
Ddecl-field.params.yaml133 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
134 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
168 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
169 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
170 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
171 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
195 … - { expr: new Float(Int.MIN_VALUE as float), val: -2.14748365E9 as float } # loss of precision
196 … - { expr: new Float(Int.MAX_VALUE as float), val: 2.14748365E9 as float } # loss of precision
197 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372E18 as float } # loss of precision
198 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372E18 as float } # loss of precision
[all …]
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets-templates/03.types/alias_unconversion/call_meth/
Dcall-meth.params.yaml133 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
134 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
168 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
169 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
170 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
171 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
195 … - { expr: new Float(Int.MIN_VALUE as float), val: -2.14748365E9 as float } # loss of precision
196 … - { expr: new Float(Int.MAX_VALUE as float), val: 2.14748365E9 as float } # loss of precision
197 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372E18 as float } # loss of precision
198 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372E18 as float } # loss of precision
[all …]
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets-templates/03.types/alias_unconversion/assn_var/
Dassn-var.params.yaml133 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
134 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
168 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
169 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
170 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
171 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
195 … - { expr: new Float(Int.MIN_VALUE as float), val: -2.14748365E9 as float } # loss of precision
196 … - { expr: new Float(Int.MAX_VALUE as float), val: 2.14748365E9 as float } # loss of precision
197 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372E18 as float } # loss of precision
198 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372E18 as float } # loss of precision
[all …]
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets-templates/03.types/alias_unconversion/decl_field/
Ddecl-field.params.yaml133 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
134 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
168 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
169 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
170 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
171 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
195 … - { expr: new Float(Int.MIN_VALUE as float), val: -2.14748365E9 as float } # loss of precision
196 … - { expr: new Float(Int.MAX_VALUE as float), val: 2.14748365E9 as float } # loss of precision
197 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372E18 as float } # loss of precision
198 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372E18 as float } # loss of precision
[all …]
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets-templates/03.types/alias_unconversion/call_cons/
Dcall-cons.params.yaml133 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
134 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
168 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
169 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
170 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
171 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
195 … - { expr: new Float(Int.MIN_VALUE as float), val: -2.14748365E9 as float } # loss of precision
196 … - { expr: new Float(Int.MAX_VALUE as float), val: 2.14748365E9 as float } # loss of precision
197 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372E18 as float } # loss of precision
198 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372E18 as float } # loss of precision
[all …]
/arkcompiler/runtime_core/static_core/plugins/ets/tests/ets-templates/03.types/alias_unconversion/decl_const/
Ddecl-const.params.yaml133 - { expr: new Int(Int.MIN_VALUE), val: -2147483600.0 as float } # loss of precision
134 - { expr: new Int(Int.MAX_VALUE), val: 2147483600.0 as float } # loss of precision
168 … - { expr: new Long(Int.MIN_VALUE as long), val: -2147483600.0 as float } # loss of precision
169 … - { expr: new Long(Int.MAX_VALUE as long), val: 2147483600.0 as float } # loss of precision
170 … - { expr: new Long(Long.MIN_VALUE), val: -9223372000000000000.0 as float } # loss of precision
171 … - { expr: new Long(Long.MAX_VALUE), val: 9223372000000000000.0 as float } # loss of precision
195 … - { expr: new Float(Int.MIN_VALUE as float), val: -2.14748365E9 as float } # loss of precision
196 … - { expr: new Float(Int.MAX_VALUE as float), val: 2.14748365E9 as float } # loss of precision
197 … - { expr: new Float(Long.MIN_VALUE as float), val: -9.223372E18 as float } # loss of precision
198 … - { expr: new Float(Long.MAX_VALUE as float), val: 9.223372E18 as float } # loss of precision
[all …]

1234567