1 /*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef BERBERIS_INTRINSICS_COMMON_INTRINSICS_H_
18 #define BERBERIS_INTRINSICS_COMMON_INTRINSICS_H_
19
20 #include <cstdint>
21
22 #include "berberis/base/checks.h"
23 #include "berberis/base/dependent_false.h"
24 #include "berberis/intrinsics/common/intrinsics_float.h" // Float16/Float32/Float64
25
26 namespace berberis {
27
28 class SIMD128Register;
29
30 namespace intrinsics {
31
32 // Value that's passed as argument of function or lambda couldn't be constexpr, but if it's
33 // passed as part of argument type then it's different.
34 // Class Value is empty, but carries the required information in its type.
35 // It can also be automatically converted into value of the specified type when needed.
36 // That way we can pass argument into a template as normal, non-template argument.
37 template <auto ValueParam>
38 class Value {
39 public:
40 using ValueType = std::remove_cvref_t<decltype(ValueParam)>;
41 static constexpr auto kValue = ValueParam;
ValueType()42 constexpr operator ValueType() const { return kValue; }
43 };
44
45 enum TemplateTypeId : uint8_t {
46 kInt8T = 1,
47 kUInt8T = 0,
48 kInt16T = 3,
49 kUInt16T = 2,
50 kInt32T = 5,
51 kUInt32T = 4,
52 kInt64T = 7,
53 kUInt64T = 6,
54 kFloat16 = 10,
55 kFloat32 = 12,
56 kFloat64 = 14,
57 kSIMD128Register = 16,
58 };
59
TemplateTypeIdToFloat(TemplateTypeId value)60 constexpr TemplateTypeId TemplateTypeIdToFloat(TemplateTypeId value) {
61 DCHECK(value >= kUInt16T && value <= kInt64T);
62 return TemplateTypeId{static_cast<uint8_t>((value & 0x6) + 8)};
63 }
64
TemplateTypeIdToInt(TemplateTypeId value)65 constexpr TemplateTypeId TemplateTypeIdToInt(TemplateTypeId value) {
66 DCHECK((value >= kFloat16 && value <= kFloat64) && !(value & 1));
67 return TemplateTypeId{static_cast<uint8_t>(value - 8)};
68 }
69
TemplateTypeIdToNarrow(TemplateTypeId value)70 constexpr TemplateTypeId TemplateTypeIdToNarrow(TemplateTypeId value) {
71 DCHECK((value >= kUInt16T && value <= kInt64T) ||
72 ((value >= kFloat32 && value <= kFloat64) && !(value & 1)));
73 return TemplateTypeId{static_cast<uint8_t>(value - 2)};
74 }
75
TemplateTypeIdToSigned(TemplateTypeId value)76 constexpr TemplateTypeId TemplateTypeIdToSigned(TemplateTypeId value) {
77 DCHECK(value <= kInt64T);
78 return TemplateTypeId{static_cast<uint8_t>(value | 1)};
79 }
80
TemplateTypeIdSizeOf(TemplateTypeId value)81 constexpr int TemplateTypeIdSizeOf(TemplateTypeId value) {
82 if (value == kSIMD128Register) {
83 return 16;
84 }
85 return 1 << ((value & 0b110) >> 1);
86 }
87
TemplateTypeIdToUnsigned(TemplateTypeId value)88 constexpr TemplateTypeId TemplateTypeIdToUnsigned(TemplateTypeId value) {
89 DCHECK(value <= kInt64T);
90 return TemplateTypeId{static_cast<uint8_t>(value & ~1)};
91 }
92
TemplateTypeIdToWide(TemplateTypeId value)93 constexpr TemplateTypeId TemplateTypeIdToWide(TemplateTypeId value) {
94 DCHECK(value <= kInt32T || ((value >= kFloat16 && value <= kFloat32) && !(value & 1)));
95 return TemplateTypeId{static_cast<uint8_t>(value + 2)};
96 }
97
98 template <typename Type>
IdFromType()99 constexpr TemplateTypeId IdFromType() {
100 if constexpr (std::is_same_v<int8_t, std::decay_t<Type>>) {
101 return TemplateTypeId::kInt8T;
102 } else if constexpr (std::is_same_v<uint8_t, std::decay_t<Type>>) {
103 return TemplateTypeId::kUInt8T;
104 } else if constexpr (std::is_same_v<int16_t, std::decay_t<Type>>) {
105 return TemplateTypeId::kInt16T;
106 } else if constexpr (std::is_same_v<uint16_t, std::decay_t<Type>>) {
107 return TemplateTypeId::kUInt16T;
108 } else if constexpr (std::is_same_v<int32_t, std::decay_t<Type>>) {
109 return TemplateTypeId::kInt32T;
110 } else if constexpr (std::is_same_v<uint32_t, std::decay_t<Type>>) {
111 return TemplateTypeId::kUInt32T;
112 } else if constexpr (std::is_same_v<int64_t, std::decay_t<Type>>) {
113 return TemplateTypeId::kInt64T;
114 } else if constexpr (std::is_same_v<uint64_t, std::decay_t<Type>>) {
115 return TemplateTypeId::kUInt64T;
116 } else if constexpr (std::is_same_v<Float16, std::decay_t<Type>>) {
117 return TemplateTypeId::kFloat16;
118 } else if constexpr (std::is_same_v<Float32, std::decay_t<Type>>) {
119 return TemplateTypeId::kFloat32;
120 } else if constexpr (std::is_same_v<Float64, std::decay_t<Type>>) {
121 return TemplateTypeId::kFloat64;
122 } else if constexpr (std::is_same_v<SIMD128Register, std::decay_t<Type>>) {
123 return TemplateTypeId::kSIMD128Register;
124 } else {
125 static_assert(kDependentTypeFalse<Type>);
126 }
127 }
128
129 template <typename Type>
130 constexpr TemplateTypeId kIdFromType = IdFromType<Type>();
131
132 constexpr TemplateTypeId IntSizeToTemplateTypeId(uint8_t size, bool is_signed = false) {
133 DCHECK(std::has_single_bit(size));
134 DCHECK(size < 16);
135 return static_cast<TemplateTypeId>((std::countr_zero(size) << 1) + is_signed);
136 }
137
138 template <enum TemplateTypeId>
139 class TypeFromIdHelper;
140
141 #pragma push_macro("DEFINE_TEMPLATE_TYPE_FROM_ENUM")
142 #undef DEFINE_TEMPLATE_TYPE_FROM_ENUM
143 #define DEFINE_TEMPLATE_TYPE_FROM_ENUM(kEnumValue, TemplateType) \
144 template <> \
145 class TypeFromIdHelper<kEnumValue> { \
146 public: \
147 using Type = TemplateType; \
148 }
149
150 DEFINE_TEMPLATE_TYPE_FROM_ENUM(kInt8T, int8_t);
151 DEFINE_TEMPLATE_TYPE_FROM_ENUM(kUInt8T, uint8_t);
152 DEFINE_TEMPLATE_TYPE_FROM_ENUM(kInt16T, int16_t);
153 DEFINE_TEMPLATE_TYPE_FROM_ENUM(kUInt16T, uint16_t);
154 DEFINE_TEMPLATE_TYPE_FROM_ENUM(kInt32T, int32_t);
155 DEFINE_TEMPLATE_TYPE_FROM_ENUM(kUInt32T, uint32_t);
156 DEFINE_TEMPLATE_TYPE_FROM_ENUM(kInt64T, int64_t);
157 DEFINE_TEMPLATE_TYPE_FROM_ENUM(kUInt64T, uint64_t);
158 DEFINE_TEMPLATE_TYPE_FROM_ENUM(kFloat16, Float16);
159 DEFINE_TEMPLATE_TYPE_FROM_ENUM(kFloat32, Float32);
160 DEFINE_TEMPLATE_TYPE_FROM_ENUM(kFloat64, Float64);
161 DEFINE_TEMPLATE_TYPE_FROM_ENUM(kSIMD128Register, SIMD128Register);
162
163 #pragma pop_macro("DEFINE_TEMPLATE_TYPE_FROM_ENUM")
164
165 template <enum TemplateTypeId kEnumValue>
166 using TypeFromId = TypeFromIdHelper<kEnumValue>::Type;
167
168 // If we carry TemplateTypeId then we can do the exact same manipulations wuth it as with
169 // normal value, but also can get actual type from it and do appropriate operations:
170 // make signed, make unsigned, widen, narrow, etc.
171 template <TemplateTypeId ValueParam>
172 class Value<ValueParam> {
173 public:
174 using Type = TypeFromId<ValueParam>;
175 using ValueType = TemplateTypeId;
176 static constexpr auto kValue = ValueParam;
TemplateTypeId()177 constexpr operator TemplateTypeId() const { return kValue; }
178 };
179
180 #pragma push_macro("DEFINE_VALUE_FUNCTION")
181 #undef DEFINE_VALUE_FUNCTION
182 #define DEFINE_VALUE_FUNCTION(FunctionName) \
183 template <TemplateTypeId ValueParam> \
184 constexpr Value<FunctionName(ValueParam)> FunctionName(Value<ValueParam>) { \
185 return {}; \
186 }
187
188 DEFINE_VALUE_FUNCTION(TemplateTypeIdToFloat)
189 DEFINE_VALUE_FUNCTION(TemplateTypeIdToInt)
190 DEFINE_VALUE_FUNCTION(TemplateTypeIdToNarrow)
191 DEFINE_VALUE_FUNCTION(TemplateTypeIdToSigned)
192 DEFINE_VALUE_FUNCTION(TemplateTypeIdSizeOf)
193 DEFINE_VALUE_FUNCTION(TemplateTypeIdToUnsigned)
194 DEFINE_VALUE_FUNCTION(TemplateTypeIdToWide)
195
196 #pragma pop_macro("DEFINE_VALUE_FUNCTION")
197
198 #pragma push_macro("DEFINE_VALUE_OPERATOR")
199 #undef DEFINE_VALUE_OPERATOR
200 #define DEFINE_VALUE_OPERATOR(operator_name) \
201 template <auto ValueParam1, auto ValueParam2> \
202 constexpr Value<(ValueParam1 operator_name ValueParam2)> operator operator_name( \
203 Value<ValueParam1>, Value<ValueParam2>) { \
204 return {}; \
205 }
206
207 DEFINE_VALUE_OPERATOR(+)
208 DEFINE_VALUE_OPERATOR(-)
209 DEFINE_VALUE_OPERATOR(*)
210 DEFINE_VALUE_OPERATOR(/)
211 DEFINE_VALUE_OPERATOR(<<)
212 DEFINE_VALUE_OPERATOR(>>)
213 DEFINE_VALUE_OPERATOR(==)
214 DEFINE_VALUE_OPERATOR(!=)
215 DEFINE_VALUE_OPERATOR(>)
216 DEFINE_VALUE_OPERATOR(<)
217 DEFINE_VALUE_OPERATOR(<=)
218 DEFINE_VALUE_OPERATOR(>=)
219 DEFINE_VALUE_OPERATOR(&&)
220 DEFINE_VALUE_OPERATOR(||)
221
222 #pragma pop_macro("DEFINE_VALUE_OPERATOR")
223
224 // A solution for the inability to call generic implementation from specialization.
225 // Declaration:
226 // template <typename Type,
227 // int size,
228 // enum PreferredIntrinsicsImplementation = kUseAssemblerImplementationIfPossible>
229 // inline std::tuple<SIMD128Register> VectorMultiplyByScalarInt(SIMD128Register op1,
230 // SIMD128Register op2);
231 // Normal use only specifies two arguments, e.g. VectorMultiplyByScalarInt<uint32_t, 2>,
232 // but assembler implementation can (if SSE 4.1 is not available) do the following call:
233 // return VectorMultiplyByScalarInt<uint32_t, 2, kUseCppImplementation>(in0, in1);
234 //
235 // Because PreferredIntrinsicsImplementation argument has non-default value we have call to the
236 // generic C-based implementation here.
237
238 enum PreferredIntrinsicsImplementation {
239 kUseAssemblerImplementationIfPossible,
240 kUseCppImplementation
241 };
242
243 } // namespace intrinsics
244
245 } // namespace berberis
246
247 #endif // BERBERIS_INTRINSICS_COMMON_INTRINSICS_H_
248