1 /*
2 * Copyright (c) 2018-2020 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #include "arm_gemm.hpp"
26
27 #include <cstdint>
28 #include <functional>
29
30 namespace arm_gemm {
31
32 /* Structure describing an implementation. For each supported combination
33 * of types, a static list of these structures is built up to describe the
34 * implementations available.
35 */
36 template<typename Top, typename Tret, class OutputStage = Nothing>
37 struct GemmImplementation {
38 const GemmMethod method;
39 const char * name;
40 std::function<bool(const GemmArgs &, const OutputStage &)> is_supported = {};
41 std::function<uint64_t(const GemmArgs &, const OutputStage &)> cycle_estimate = {};
42 std::function<GemmCommon<Top, Tret> *(const GemmArgs &, const OutputStage &)> instantiate = {};
43
do_is_supportedarm_gemm::GemmImplementation44 bool do_is_supported(const GemmArgs &args, const OutputStage &os) const {
45 if (is_supported != nullptr) {
46 return is_supported(args, os);
47 } else {
48 return true;
49 }
50 }
51
do_cycle_estimatearm_gemm::GemmImplementation52 uint64_t do_cycle_estimate(const GemmArgs &args, const OutputStage &os) const {
53 if (cycle_estimate != nullptr) {
54 return cycle_estimate(args, os);
55 } else {
56 return 0;
57 }
58 }
59
do_instantiatearm_gemm::GemmImplementation60 GemmCommon<Top, Tret> *do_instantiate(const GemmArgs &args, const OutputStage &os) const {
61 return instantiate(args, os);
62 }
63
64 GemmImplementation(const GemmImplementation &) = default;
65 GemmImplementation & operator= (const GemmImplementation &) = default;
66
GemmImplementationarm_gemm::GemmImplementation67 GemmImplementation(GemmMethod m, const char *n,
68 std::function<bool(const GemmArgs &, const OutputStage &)> is_supported, std::function<bool(const GemmArgs &, const OutputStage &)> is_recommended,
69 std::function<GemmCommon<Top, Tret> *(const GemmArgs &, const OutputStage &)> instantiate) :
70 method(m), name(n), is_supported(is_supported),
71 cycle_estimate( [is_recommended](const GemmArgs &args, const OutputStage &os) { return (is_recommended == nullptr) ? 0 : (is_recommended(args, os) ? 0 : UINT64_MAX); } ),
72 instantiate(instantiate) { }
73 };
74
75 /* Slightly different version of above for straightforward GEMMs with no
76 * output stage, so the std::functions there don't have to deal with the
77 * unnecessary second argument. */
78 template<typename Top, typename Tret>
79 struct GemmImplementation<Top, Tret, Nothing> {
80 const GemmMethod method;
81 const char * name;
82 std::function<bool(const GemmArgs &)> is_supported = {};
83 std::function<uint64_t(const GemmArgs &)> cycle_estimate = {};
84 std::function<GemmCommon<Top, Tret> *(const GemmArgs &)> instantiate = {};
85
do_is_supportedarm_gemm::GemmImplementation86 bool do_is_supported(const GemmArgs &args, const Nothing &) const {
87 if (is_supported != nullptr) {
88 return is_supported(args);
89 } else {
90 return true;
91 }
92 }
93
do_cycle_estimatearm_gemm::GemmImplementation94 uint64_t do_cycle_estimate(const GemmArgs &args, const Nothing &) const {
95 if (cycle_estimate != nullptr) {
96 return cycle_estimate(args);
97 } else {
98 return 0;
99 }
100 }
101
do_instantiatearm_gemm::GemmImplementation102 GemmCommon<Top, Tret> *do_instantiate(const GemmArgs &args, const Nothing &) const {
103 return instantiate(args);
104 }
105
with_estimatearm_gemm::GemmImplementation106 static GemmImplementation with_estimate(GemmMethod m, const char *n,
107 std::function<bool(const GemmArgs &)> is_supported, std::function<uint64_t(const GemmArgs &)> cycle_estimate,
108 std::function<GemmCommon<Top, Tret> *(const GemmArgs &)> instantiate) {
109 GemmImplementation impl(m,n);
110
111 impl.is_supported=is_supported;
112 impl.cycle_estimate=cycle_estimate;
113 impl.instantiate=instantiate;
114
115 return impl;
116 }
117
118 GemmImplementation(const GemmImplementation &) = default;
119 GemmImplementation & operator= (const GemmImplementation &) = default;
120
GemmImplementationarm_gemm::GemmImplementation121 GemmImplementation(GemmMethod m, const char * n) : method(m), name(n) {}
122
GemmImplementationarm_gemm::GemmImplementation123 GemmImplementation(GemmMethod m, const char *n,
124 std::function<bool(const GemmArgs &)> is_supported, std::function<bool(const GemmArgs &)> is_recommended,
125 std::function<GemmCommon<Top, Tret> *(const GemmArgs &)> instantiate) :
126 method(m), name(n), is_supported(is_supported),
127 cycle_estimate( [is_recommended](const GemmArgs &args) -> uint64_t { return (is_recommended == nullptr) ? 0 : (is_recommended(args) ? 0 : UINT64_MAX); } ),
128 instantiate(instantiate) { }
129 };
130
131 /* "Master" function implemented for each valid combination of types.
132 * Returns a list of GEMM implementation descriptors for processing by the
133 * other functions, terminated by an implementation with
134 * method==GemmMethod::DEFAULT. */
135 template<typename Top, typename Tret, class OutputStage = Nothing>
136 const GemmImplementation<Top, Tret, OutputStage> *gemm_implementation_list();
137
138 /*
139 * Select a GEMM implementation for the given arguments.
140 *
141 * The logic here returns the method on the list which supports the
142 * requested problem parameters, matches the provided filters (method and/or
143 * name string match) and offers the lowest cycle estimate. A cycle
144 * estimate of '0' is treated as a special value, causing the corresponding
145 * method to be selected immediately.
146 *
147 * If no method supports the requested parameters and passes the filters,
148 * this function returns false and doesn't touch the provided pointer
149 * reference.
150 */
151 template<typename Top, typename Tret, class OutputStage>
find_implementation(const GemmArgs & args,const OutputStage & os,const GemmImplementation<Top,Tret,OutputStage> * & impl)152 bool find_implementation(const GemmArgs &args, const OutputStage &os, const GemmImplementation<Top, Tret, OutputStage> * &impl) {
153 auto gemms = gemm_implementation_list<Top, Tret, OutputStage>();
154 const GemmConfig *cfg = args._cfg;
155
156 const GemmImplementation<Top, Tret, OutputStage> *saved_impl = nullptr;
157 uint64_t best_estimate = 0;
158
159 for (const GemmImplementation<Top, Tret, OutputStage> *i = gemms; i->method != GemmMethod::DEFAULT; i++) {
160 /* Skip if this implementation doesn't support these args. */
161 if (!i->do_is_supported(args, os)) {
162 continue;
163 }
164
165 /* Skip if a specific method is requested and this is a different one. */
166 if (cfg && cfg->method != GemmMethod::DEFAULT && i->method != cfg->method) {
167 continue;
168 }
169
170 /* Skip if a filter is to be applied and it doesn't match. */
171 if (cfg && cfg->filter != "" && !strstr(i->name, cfg->filter.c_str())) {
172 continue;
173 }
174
175 /* Test the cycle estimate */
176 uint64_t estimate = i->do_cycle_estimate(args, os);
177
178 /* Short circuit - if the estimate is zero, return this one immediately. */
179 if (estimate==0) {
180 impl=i;
181 return true;
182 }
183
184 /* Otherwise, remember this is our best so far if we don't yet have
185 * a valid candidate, or we beat the estimate. */
186 if ((saved_impl == nullptr) || (estimate < best_estimate)) {
187 saved_impl = i;
188 best_estimate = estimate;
189 }
190 }
191
192 /* Return whichever method gave the best estimate. */
193 if (saved_impl != nullptr) {
194 impl = saved_impl;
195 return true;
196 }
197
198 return false;
199 }
200
201 template<typename Top, typename Tret, class OutputStage>
get_compatible_kernels(const GemmArgs & args,const OutputStage & os)202 std::vector<KernelDescription> get_compatible_kernels(const GemmArgs &args, const OutputStage &os) {
203 std::vector<KernelDescription> res;
204
205 /* Find out what the default implementation in so we can set the flag accordingly later. */
206 const GemmImplementation<Top, Tret, OutputStage> *default_impl;
207 find_implementation(args, os, default_impl);
208
209 auto gemms = gemm_implementation_list<Top, Tret, OutputStage>();
210
211 for (const GemmImplementation<Top, Tret, OutputStage> *i = gemms; i->method != GemmMethod::DEFAULT; i++) {
212 /* Check that this implementation supports the presented problem. */
213
214 if (!i->do_is_supported(args, os)) {
215 continue;
216 }
217
218 res.push_back(KernelDescription(i->method, i->name, i==default_impl, i->do_cycle_estimate(args, os)));
219 }
220
221 return res;
222 }
223
224 template<typename Top, typename Tret, class OutputStage>
gemm(const GemmArgs & args,const OutputStage & os)225 UniqueGemmCommon<Top, Tret> gemm(const GemmArgs &args, const OutputStage &os) {
226 const GemmImplementation<Top, Tret, OutputStage> *impl;
227
228 if (find_implementation<Top, Tret, OutputStage>(args, os, impl)) {
229 return UniqueGemmCommon<Top, Tret>(impl->do_instantiate(args, os));
230 }
231
232 return UniqueGemmCommon<Top, Tret>(nullptr);
233 }
234
235 template<typename Top, typename Tret, class OutputStage>
get_gemm_method(const GemmArgs & args,const OutputStage & os)236 KernelDescription get_gemm_method(const GemmArgs &args, const OutputStage &os) {
237 const GemmImplementation<Top, Tret, OutputStage> *impl;
238
239 if (find_implementation<Top, Tret>(args, os, impl)) {
240 return KernelDescription(impl->method, impl->name);
241 }
242
243 /* This shouldn't happen - there should always be at least one valid implementation. */
244 return KernelDescription();
245 }
246
247 } // namespace arm_gemm
248