1 // Copyright (c) 2018 Google LLC.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "source/opt/loop_dependence.h"
16
17 #include <functional>
18 #include <numeric>
19 #include <string>
20 #include <utility>
21 #include <vector>
22
23 #include "source/opt/instruction.h"
24 #include "source/opt/scalar_analysis_nodes.h"
25
26 namespace spvtools {
27 namespace opt {
28
29 using SubscriptPair = std::pair<SENode*, SENode*>;
30
31 namespace {
32
33 // Calculate the greatest common divisor of a & b using Stein's algorithm.
34 // https://en.wikipedia.org/wiki/Binary_GCD_algorithm
GreatestCommonDivisor(int64_t a,int64_t b)35 int64_t GreatestCommonDivisor(int64_t a, int64_t b) {
36 // Simple cases
37 if (a == b) {
38 return a;
39 } else if (a == 0) {
40 return b;
41 } else if (b == 0) {
42 return a;
43 }
44
45 // Both even
46 if (a % 2 == 0 && b % 2 == 0) {
47 return 2 * GreatestCommonDivisor(a / 2, b / 2);
48 }
49
50 // Even a, odd b
51 if (a % 2 == 0 && b % 2 == 1) {
52 return GreatestCommonDivisor(a / 2, b);
53 }
54
55 // Odd a, even b
56 if (a % 2 == 1 && b % 2 == 0) {
57 return GreatestCommonDivisor(a, b / 2);
58 }
59
60 // Both odd, reduce the larger argument
61 if (a > b) {
62 return GreatestCommonDivisor((a - b) / 2, b);
63 } else {
64 return GreatestCommonDivisor((b - a) / 2, a);
65 }
66 }
67
68 // Check if node is affine, ie in the form: a0*i0 + a1*i1 + ... an*in + c
69 // and contains only the following types of nodes: SERecurrentNode, SEAddNode
70 // and SEConstantNode
IsInCorrectFormForGCDTest(SENode * node)71 bool IsInCorrectFormForGCDTest(SENode* node) {
72 bool children_ok = true;
73
74 if (auto add_node = node->AsSEAddNode()) {
75 for (auto child : add_node->GetChildren()) {
76 children_ok &= IsInCorrectFormForGCDTest(child);
77 }
78 }
79
80 bool this_ok = node->AsSERecurrentNode() || node->AsSEAddNode() ||
81 node->AsSEConstantNode();
82
83 return children_ok && this_ok;
84 }
85
86 // If |node| is an SERecurrentNode then returns |node| or if |node| is an
87 // SEAddNode returns a vector of SERecurrentNode that are its children.
GetAllTopLevelRecurrences(SENode * node)88 std::vector<SERecurrentNode*> GetAllTopLevelRecurrences(SENode* node) {
89 auto nodes = std::vector<SERecurrentNode*>{};
90 if (auto recurrent_node = node->AsSERecurrentNode()) {
91 nodes.push_back(recurrent_node);
92 }
93
94 if (auto add_node = node->AsSEAddNode()) {
95 for (auto child : add_node->GetChildren()) {
96 auto child_nodes = GetAllTopLevelRecurrences(child);
97 nodes.insert(nodes.end(), child_nodes.begin(), child_nodes.end());
98 }
99 }
100
101 return nodes;
102 }
103
104 // If |node| is an SEConstantNode then returns |node| or if |node| is an
105 // SEAddNode returns a vector of SEConstantNode that are its children.
GetAllTopLevelConstants(SENode * node)106 std::vector<SEConstantNode*> GetAllTopLevelConstants(SENode* node) {
107 auto nodes = std::vector<SEConstantNode*>{};
108 if (auto recurrent_node = node->AsSEConstantNode()) {
109 nodes.push_back(recurrent_node);
110 }
111
112 if (auto add_node = node->AsSEAddNode()) {
113 for (auto child : add_node->GetChildren()) {
114 auto child_nodes = GetAllTopLevelConstants(child);
115 nodes.insert(nodes.end(), child_nodes.begin(), child_nodes.end());
116 }
117 }
118
119 return nodes;
120 }
121
AreOffsetsAndCoefficientsConstant(const std::vector<SERecurrentNode * > & nodes)122 bool AreOffsetsAndCoefficientsConstant(
123 const std::vector<SERecurrentNode*>& nodes) {
124 for (auto node : nodes) {
125 if (!node->GetOffset()->AsSEConstantNode() ||
126 !node->GetOffset()->AsSEConstantNode()) {
127 return false;
128 }
129 }
130 return true;
131 }
132
133 // Fold all SEConstantNode that appear in |recurrences| and |constants| into a
134 // single integer value.
CalculateConstantTerm(const std::vector<SERecurrentNode * > & recurrences,const std::vector<SEConstantNode * > & constants)135 int64_t CalculateConstantTerm(const std::vector<SERecurrentNode*>& recurrences,
136 const std::vector<SEConstantNode*>& constants) {
137 int64_t constant_term = 0;
138 for (auto recurrence : recurrences) {
139 constant_term +=
140 recurrence->GetOffset()->AsSEConstantNode()->FoldToSingleValue();
141 }
142
143 for (auto constant : constants) {
144 constant_term += constant->FoldToSingleValue();
145 }
146
147 return constant_term;
148 }
149
CalculateGCDFromCoefficients(const std::vector<SERecurrentNode * > & recurrences,int64_t running_gcd)150 int64_t CalculateGCDFromCoefficients(
151 const std::vector<SERecurrentNode*>& recurrences, int64_t running_gcd) {
152 for (SERecurrentNode* recurrence : recurrences) {
153 auto coefficient = recurrence->GetCoefficient()->AsSEConstantNode();
154
155 running_gcd = GreatestCommonDivisor(
156 running_gcd, std::abs(coefficient->FoldToSingleValue()));
157 }
158
159 return running_gcd;
160 }
161
162 // Compare 2 fractions while first normalizing them, e.g. 2/4 and 4/8 will both
163 // be simplified to 1/2 and then determined to be equal.
NormalizeAndCompareFractions(int64_t numerator_0,int64_t denominator_0,int64_t numerator_1,int64_t denominator_1)164 bool NormalizeAndCompareFractions(int64_t numerator_0, int64_t denominator_0,
165 int64_t numerator_1, int64_t denominator_1) {
166 auto gcd_0 =
167 GreatestCommonDivisor(std::abs(numerator_0), std::abs(denominator_0));
168 auto gcd_1 =
169 GreatestCommonDivisor(std::abs(numerator_1), std::abs(denominator_1));
170
171 auto normalized_numerator_0 = numerator_0 / gcd_0;
172 auto normalized_denominator_0 = denominator_0 / gcd_0;
173 auto normalized_numerator_1 = numerator_1 / gcd_1;
174 auto normalized_denominator_1 = denominator_1 / gcd_1;
175
176 return normalized_numerator_0 == normalized_numerator_1 &&
177 normalized_denominator_0 == normalized_denominator_1;
178 }
179
180 } // namespace
181
GetDependence(const Instruction * source,const Instruction * destination,DistanceVector * distance_vector)182 bool LoopDependenceAnalysis::GetDependence(const Instruction* source,
183 const Instruction* destination,
184 DistanceVector* distance_vector) {
185 // Start off by finding and marking all the loops in |loops_| that are
186 // irrelevant to the dependence analysis.
187 MarkUnsusedDistanceEntriesAsIrrelevant(source, destination, distance_vector);
188
189 Instruction* source_access_chain = GetOperandDefinition(source, 0);
190 Instruction* destination_access_chain = GetOperandDefinition(destination, 0);
191
192 auto num_access_chains =
193 (source_access_chain->opcode() == spv::Op::OpAccessChain) +
194 (destination_access_chain->opcode() == spv::Op::OpAccessChain);
195
196 // If neither is an access chain, then they are load/store to a variable.
197 if (num_access_chains == 0) {
198 if (source_access_chain != destination_access_chain) {
199 // Not the same location, report independence
200 return true;
201 } else {
202 // Accessing the same variable
203 for (auto& entry : distance_vector->GetEntries()) {
204 entry = DistanceEntry();
205 }
206 return false;
207 }
208 }
209
210 // If only one is an access chain, it could be accessing a part of a struct
211 if (num_access_chains == 1) {
212 auto source_is_chain =
213 source_access_chain->opcode() == spv::Op::OpAccessChain;
214 auto access_chain =
215 source_is_chain ? source_access_chain : destination_access_chain;
216 auto variable =
217 source_is_chain ? destination_access_chain : source_access_chain;
218
219 auto location_in_chain = GetOperandDefinition(access_chain, 0);
220
221 if (variable != location_in_chain) {
222 // Not the same location, report independence
223 return true;
224 } else {
225 // Accessing the same variable
226 for (auto& entry : distance_vector->GetEntries()) {
227 entry = DistanceEntry();
228 }
229 return false;
230 }
231 }
232
233 // If the access chains aren't collecting from the same structure there is no
234 // dependence.
235 Instruction* source_array = GetOperandDefinition(source_access_chain, 0);
236 Instruction* destination_array =
237 GetOperandDefinition(destination_access_chain, 0);
238
239 // Nested access chains are not supported yet, bail out.
240 if (source_array->opcode() == spv::Op::OpAccessChain ||
241 destination_array->opcode() == spv::Op::OpAccessChain) {
242 for (auto& entry : distance_vector->GetEntries()) {
243 entry = DistanceEntry();
244 }
245 return false;
246 }
247
248 if (source_array != destination_array) {
249 PrintDebug("Proved independence through different arrays.");
250 return true;
251 }
252
253 // To handle multiple subscripts we must get every operand in the access
254 // chains past the first.
255 std::vector<Instruction*> source_subscripts = GetSubscripts(source);
256 std::vector<Instruction*> destination_subscripts = GetSubscripts(destination);
257
258 auto sets_of_subscripts =
259 PartitionSubscripts(source_subscripts, destination_subscripts);
260
261 auto first_coupled = std::partition(
262 std::begin(sets_of_subscripts), std::end(sets_of_subscripts),
263 [](const std::set<std::pair<Instruction*, Instruction*>>& set) {
264 return set.size() == 1;
265 });
266
267 // Go through each subscript testing for independence.
268 // If any subscript results in independence, we prove independence between the
269 // load and store.
270 // If we can't prove independence we store what information we can gather in
271 // a DistanceVector.
272 for (auto it = std::begin(sets_of_subscripts); it < first_coupled; ++it) {
273 auto source_subscript = std::get<0>(*(*it).begin());
274 auto destination_subscript = std::get<1>(*(*it).begin());
275
276 SENode* source_node = scalar_evolution_.SimplifyExpression(
277 scalar_evolution_.AnalyzeInstruction(source_subscript));
278 SENode* destination_node = scalar_evolution_.SimplifyExpression(
279 scalar_evolution_.AnalyzeInstruction(destination_subscript));
280
281 // Check the loops are in a form we support.
282 auto subscript_pair = std::make_pair(source_node, destination_node);
283
284 const Loop* loop = GetLoopForSubscriptPair(subscript_pair);
285 if (loop) {
286 if (!IsSupportedLoop(loop)) {
287 PrintDebug(
288 "GetDependence found an unsupported loop form. Assuming <=> for "
289 "loop.");
290 DistanceEntry* distance_entry =
291 GetDistanceEntryForSubscriptPair(subscript_pair, distance_vector);
292 if (distance_entry) {
293 distance_entry->direction = DistanceEntry::Directions::ALL;
294 }
295 continue;
296 }
297 }
298
299 // If either node is simplified to a CanNotCompute we can't perform any
300 // analysis so must assume <=> dependence and return.
301 if (source_node->GetType() == SENode::CanNotCompute ||
302 destination_node->GetType() == SENode::CanNotCompute) {
303 // Record the <=> dependence if we can get a DistanceEntry
304 PrintDebug(
305 "GetDependence found source_node || destination_node as "
306 "CanNotCompute. Abandoning evaluation for this subscript.");
307 DistanceEntry* distance_entry =
308 GetDistanceEntryForSubscriptPair(subscript_pair, distance_vector);
309 if (distance_entry) {
310 distance_entry->direction = DistanceEntry::Directions::ALL;
311 }
312 continue;
313 }
314
315 // We have no induction variables so can apply a ZIV test.
316 if (IsZIV(subscript_pair)) {
317 PrintDebug("Found a ZIV subscript pair");
318 if (ZIVTest(subscript_pair)) {
319 PrintDebug("Proved independence with ZIVTest.");
320 return true;
321 }
322 }
323
324 // We have only one induction variable so should attempt an SIV test.
325 if (IsSIV(subscript_pair)) {
326 PrintDebug("Found a SIV subscript pair.");
327 if (SIVTest(subscript_pair, distance_vector)) {
328 PrintDebug("Proved independence with SIVTest.");
329 return true;
330 }
331 }
332
333 // We have multiple induction variables so should attempt an MIV test.
334 if (IsMIV(subscript_pair)) {
335 PrintDebug("Found a MIV subscript pair.");
336 if (GCDMIVTest(subscript_pair)) {
337 PrintDebug("Proved independence with the GCD test.");
338 auto current_loops = CollectLoops(source_node, destination_node);
339
340 for (auto current_loop : current_loops) {
341 auto distance_entry =
342 GetDistanceEntryForLoop(current_loop, distance_vector);
343 distance_entry->direction = DistanceEntry::Directions::NONE;
344 }
345 return true;
346 }
347 }
348 }
349
350 for (auto it = first_coupled; it < std::end(sets_of_subscripts); ++it) {
351 auto coupled_instructions = *it;
352 std::vector<SubscriptPair> coupled_subscripts{};
353
354 for (const auto& elem : coupled_instructions) {
355 auto source_subscript = std::get<0>(elem);
356 auto destination_subscript = std::get<1>(elem);
357
358 SENode* source_node = scalar_evolution_.SimplifyExpression(
359 scalar_evolution_.AnalyzeInstruction(source_subscript));
360 SENode* destination_node = scalar_evolution_.SimplifyExpression(
361 scalar_evolution_.AnalyzeInstruction(destination_subscript));
362
363 coupled_subscripts.push_back({source_node, destination_node});
364 }
365
366 auto supported = true;
367
368 for (const auto& subscript : coupled_subscripts) {
369 auto loops = CollectLoops(std::get<0>(subscript), std::get<1>(subscript));
370
371 auto is_subscript_supported =
372 std::all_of(std::begin(loops), std::end(loops),
373 [this](const Loop* l) { return IsSupportedLoop(l); });
374
375 supported = supported && is_subscript_supported;
376 }
377
378 if (DeltaTest(coupled_subscripts, distance_vector)) {
379 return true;
380 }
381 }
382
383 // We were unable to prove independence so must gather all of the direction
384 // information we found.
385 PrintDebug(
386 "Couldn't prove independence.\n"
387 "All possible direction information has been collected in the input "
388 "DistanceVector.");
389
390 return false;
391 }
392
ZIVTest(const std::pair<SENode *,SENode * > & subscript_pair)393 bool LoopDependenceAnalysis::ZIVTest(
394 const std::pair<SENode*, SENode*>& subscript_pair) {
395 auto source = std::get<0>(subscript_pair);
396 auto destination = std::get<1>(subscript_pair);
397
398 PrintDebug("Performing ZIVTest");
399 // If source == destination, dependence with direction = and distance 0.
400 if (source == destination) {
401 PrintDebug("ZIVTest found EQ dependence.");
402 return false;
403 } else {
404 PrintDebug("ZIVTest found independence.");
405 // Otherwise we prove independence.
406 return true;
407 }
408 }
409
SIVTest(const std::pair<SENode *,SENode * > & subscript_pair,DistanceVector * distance_vector)410 bool LoopDependenceAnalysis::SIVTest(
411 const std::pair<SENode*, SENode*>& subscript_pair,
412 DistanceVector* distance_vector) {
413 DistanceEntry* distance_entry =
414 GetDistanceEntryForSubscriptPair(subscript_pair, distance_vector);
415 if (!distance_entry) {
416 PrintDebug(
417 "SIVTest could not find a DistanceEntry for subscript_pair. Exiting");
418 }
419
420 SENode* source_node = std::get<0>(subscript_pair);
421 SENode* destination_node = std::get<1>(subscript_pair);
422
423 int64_t source_induction_count = CountInductionVariables(source_node);
424 int64_t destination_induction_count =
425 CountInductionVariables(destination_node);
426
427 // If the source node has no induction variables we can apply a
428 // WeakZeroSrcTest.
429 if (source_induction_count == 0) {
430 PrintDebug("Found source has no induction variable.");
431 if (WeakZeroSourceSIVTest(
432 source_node, destination_node->AsSERecurrentNode(),
433 destination_node->AsSERecurrentNode()->GetCoefficient(),
434 distance_entry)) {
435 PrintDebug("Proved independence with WeakZeroSourceSIVTest.");
436 distance_entry->dependence_information =
437 DistanceEntry::DependenceInformation::DIRECTION;
438 distance_entry->direction = DistanceEntry::Directions::NONE;
439 return true;
440 }
441 }
442
443 // If the destination has no induction variables we can apply a
444 // WeakZeroDestTest.
445 if (destination_induction_count == 0) {
446 PrintDebug("Found destination has no induction variable.");
447 if (WeakZeroDestinationSIVTest(
448 source_node->AsSERecurrentNode(), destination_node,
449 source_node->AsSERecurrentNode()->GetCoefficient(),
450 distance_entry)) {
451 PrintDebug("Proved independence with WeakZeroDestinationSIVTest.");
452 distance_entry->dependence_information =
453 DistanceEntry::DependenceInformation::DIRECTION;
454 distance_entry->direction = DistanceEntry::Directions::NONE;
455 return true;
456 }
457 }
458
459 // We now need to collect the SERecurrentExpr nodes from source and
460 // destination. We do not handle cases where source or destination have
461 // multiple SERecurrentExpr nodes.
462 std::vector<SERecurrentNode*> source_recurrent_nodes =
463 source_node->CollectRecurrentNodes();
464 std::vector<SERecurrentNode*> destination_recurrent_nodes =
465 destination_node->CollectRecurrentNodes();
466
467 if (source_recurrent_nodes.size() == 1 &&
468 destination_recurrent_nodes.size() == 1) {
469 PrintDebug("Found source and destination have 1 induction variable.");
470 SERecurrentNode* source_recurrent_expr = *source_recurrent_nodes.begin();
471 SERecurrentNode* destination_recurrent_expr =
472 *destination_recurrent_nodes.begin();
473
474 // If the coefficients are identical we can apply a StrongSIVTest.
475 if (source_recurrent_expr->GetCoefficient() ==
476 destination_recurrent_expr->GetCoefficient()) {
477 PrintDebug("Found source and destination share coefficient.");
478 if (StrongSIVTest(source_node, destination_node,
479 source_recurrent_expr->GetCoefficient(),
480 distance_entry)) {
481 PrintDebug("Proved independence with StrongSIVTest");
482 distance_entry->dependence_information =
483 DistanceEntry::DependenceInformation::DIRECTION;
484 distance_entry->direction = DistanceEntry::Directions::NONE;
485 return true;
486 }
487 }
488
489 // If the coefficients are of equal magnitude and opposite sign we can
490 // apply a WeakCrossingSIVTest.
491 if (source_recurrent_expr->GetCoefficient() ==
492 scalar_evolution_.CreateNegation(
493 destination_recurrent_expr->GetCoefficient())) {
494 PrintDebug("Found source coefficient = -destination coefficient.");
495 if (WeakCrossingSIVTest(source_node, destination_node,
496 source_recurrent_expr->GetCoefficient(),
497 distance_entry)) {
498 PrintDebug("Proved independence with WeakCrossingSIVTest");
499 distance_entry->dependence_information =
500 DistanceEntry::DependenceInformation::DIRECTION;
501 distance_entry->direction = DistanceEntry::Directions::NONE;
502 return true;
503 }
504 }
505 }
506
507 return false;
508 }
509
StrongSIVTest(SENode * source,SENode * destination,SENode * coefficient,DistanceEntry * distance_entry)510 bool LoopDependenceAnalysis::StrongSIVTest(SENode* source, SENode* destination,
511 SENode* coefficient,
512 DistanceEntry* distance_entry) {
513 PrintDebug("Performing StrongSIVTest.");
514 // If both source and destination are SERecurrentNodes we can perform tests
515 // based on distance.
516 // If either source or destination contain value unknown nodes or if one or
517 // both are not SERecurrentNodes we must attempt a symbolic test.
518 std::vector<SEValueUnknown*> source_value_unknown_nodes =
519 source->CollectValueUnknownNodes();
520 std::vector<SEValueUnknown*> destination_value_unknown_nodes =
521 destination->CollectValueUnknownNodes();
522 if (source_value_unknown_nodes.size() > 0 ||
523 destination_value_unknown_nodes.size() > 0) {
524 PrintDebug(
525 "StrongSIVTest found symbolics. Will attempt SymbolicStrongSIVTest.");
526 return SymbolicStrongSIVTest(source, destination, coefficient,
527 distance_entry);
528 }
529
530 if (!source->AsSERecurrentNode() || !destination->AsSERecurrentNode()) {
531 PrintDebug(
532 "StrongSIVTest could not simplify source and destination to "
533 "SERecurrentNodes so will exit.");
534 distance_entry->direction = DistanceEntry::Directions::ALL;
535 return false;
536 }
537
538 // Build an SENode for distance.
539 std::pair<SENode*, SENode*> subscript_pair =
540 std::make_pair(source, destination);
541 const Loop* subscript_loop = GetLoopForSubscriptPair(subscript_pair);
542 SENode* source_constant_term =
543 GetConstantTerm(subscript_loop, source->AsSERecurrentNode());
544 SENode* destination_constant_term =
545 GetConstantTerm(subscript_loop, destination->AsSERecurrentNode());
546 if (!source_constant_term || !destination_constant_term) {
547 PrintDebug(
548 "StrongSIVTest could not collect the constant terms of either source "
549 "or destination so will exit.");
550 return false;
551 }
552 SENode* constant_term_delta =
553 scalar_evolution_.SimplifyExpression(scalar_evolution_.CreateSubtraction(
554 destination_constant_term, source_constant_term));
555
556 // Scalar evolution doesn't perform division, so we must fold to constants and
557 // do it manually.
558 // We must check the offset delta and coefficient are constants.
559 int64_t distance = 0;
560 SEConstantNode* delta_constant = constant_term_delta->AsSEConstantNode();
561 SEConstantNode* coefficient_constant = coefficient->AsSEConstantNode();
562 if (delta_constant && coefficient_constant) {
563 int64_t delta_value = delta_constant->FoldToSingleValue();
564 int64_t coefficient_value = coefficient_constant->FoldToSingleValue();
565 PrintDebug(
566 "StrongSIVTest found delta value and coefficient value as constants "
567 "with values:\n"
568 "\tdelta value: " +
569 ToString(delta_value) +
570 "\n\tcoefficient value: " + ToString(coefficient_value) + "\n");
571 // Check if the distance is not integral to try to prove independence.
572 if (delta_value % coefficient_value != 0) {
573 PrintDebug(
574 "StrongSIVTest proved independence through distance not being an "
575 "integer.");
576 distance_entry->dependence_information =
577 DistanceEntry::DependenceInformation::DIRECTION;
578 distance_entry->direction = DistanceEntry::Directions::NONE;
579 return true;
580 } else {
581 distance = delta_value / coefficient_value;
582 PrintDebug("StrongSIV test found distance as " + ToString(distance));
583 }
584 } else {
585 // If we can't fold delta and coefficient to single values we can't produce
586 // distance.
587 // As a result we can't perform the rest of the pass and must assume
588 // dependence in all directions.
589 PrintDebug("StrongSIVTest could not produce a distance. Must exit.");
590 distance_entry->distance = DistanceEntry::Directions::ALL;
591 return false;
592 }
593
594 // Next we gather the upper and lower bounds as constants if possible. If
595 // distance > upper_bound - lower_bound we prove independence.
596 SENode* lower_bound = GetLowerBound(subscript_loop);
597 SENode* upper_bound = GetUpperBound(subscript_loop);
598 if (lower_bound && upper_bound) {
599 PrintDebug("StrongSIVTest found bounds.");
600 SENode* bounds = scalar_evolution_.SimplifyExpression(
601 scalar_evolution_.CreateSubtraction(upper_bound, lower_bound));
602
603 if (bounds->GetType() == SENode::SENodeType::Constant) {
604 int64_t bounds_value = bounds->AsSEConstantNode()->FoldToSingleValue();
605 PrintDebug(
606 "StrongSIVTest found upper_bound - lower_bound as a constant with "
607 "value " +
608 ToString(bounds_value));
609
610 // If the absolute value of the distance is > upper bound - lower bound
611 // then we prove independence.
612 if (llabs(distance) > llabs(bounds_value)) {
613 PrintDebug(
614 "StrongSIVTest proved independence through distance escaping the "
615 "loop bounds.");
616 distance_entry->dependence_information =
617 DistanceEntry::DependenceInformation::DISTANCE;
618 distance_entry->direction = DistanceEntry::Directions::NONE;
619 distance_entry->distance = distance;
620 return true;
621 }
622 }
623 } else {
624 PrintDebug("StrongSIVTest was unable to gather lower and upper bounds.");
625 }
626
627 // Otherwise we can get a direction as follows
628 // { < if distance > 0
629 // direction = { = if distance == 0
630 // { > if distance < 0
631 PrintDebug(
632 "StrongSIVTest could not prove independence. Gathering direction "
633 "information.");
634 if (distance > 0) {
635 distance_entry->dependence_information =
636 DistanceEntry::DependenceInformation::DISTANCE;
637 distance_entry->direction = DistanceEntry::Directions::LT;
638 distance_entry->distance = distance;
639 return false;
640 }
641 if (distance == 0) {
642 distance_entry->dependence_information =
643 DistanceEntry::DependenceInformation::DISTANCE;
644 distance_entry->direction = DistanceEntry::Directions::EQ;
645 distance_entry->distance = 0;
646 return false;
647 }
648 if (distance < 0) {
649 distance_entry->dependence_information =
650 DistanceEntry::DependenceInformation::DISTANCE;
651 distance_entry->direction = DistanceEntry::Directions::GT;
652 distance_entry->distance = distance;
653 return false;
654 }
655
656 // We were unable to prove independence or discern any additional information
657 // Must assume <=> direction.
658 PrintDebug(
659 "StrongSIVTest was unable to determine any dependence information.");
660 distance_entry->direction = DistanceEntry::Directions::ALL;
661 return false;
662 }
663
SymbolicStrongSIVTest(SENode * source,SENode * destination,SENode * coefficient,DistanceEntry * distance_entry)664 bool LoopDependenceAnalysis::SymbolicStrongSIVTest(
665 SENode* source, SENode* destination, SENode* coefficient,
666 DistanceEntry* distance_entry) {
667 PrintDebug("Performing SymbolicStrongSIVTest.");
668 SENode* source_destination_delta = scalar_evolution_.SimplifyExpression(
669 scalar_evolution_.CreateSubtraction(source, destination));
670 // By cancelling out the induction variables by subtracting the source and
671 // destination we can produce an expression of symbolics and constants. This
672 // expression can be compared to the loop bounds to find if the offset is
673 // outwith the bounds.
674 std::pair<SENode*, SENode*> subscript_pair =
675 std::make_pair(source, destination);
676 const Loop* subscript_loop = GetLoopForSubscriptPair(subscript_pair);
677 if (IsProvablyOutsideOfLoopBounds(subscript_loop, source_destination_delta,
678 coefficient)) {
679 PrintDebug(
680 "SymbolicStrongSIVTest proved independence through loop bounds.");
681 distance_entry->dependence_information =
682 DistanceEntry::DependenceInformation::DIRECTION;
683 distance_entry->direction = DistanceEntry::Directions::NONE;
684 return true;
685 }
686 // We were unable to prove independence or discern any additional information.
687 // Must assume <=> direction.
688 PrintDebug(
689 "SymbolicStrongSIVTest was unable to determine any dependence "
690 "information.");
691 distance_entry->direction = DistanceEntry::Directions::ALL;
692 return false;
693 }
694
WeakZeroSourceSIVTest(SENode * source,SERecurrentNode * destination,SENode * coefficient,DistanceEntry * distance_entry)695 bool LoopDependenceAnalysis::WeakZeroSourceSIVTest(
696 SENode* source, SERecurrentNode* destination, SENode* coefficient,
697 DistanceEntry* distance_entry) {
698 PrintDebug("Performing WeakZeroSourceSIVTest.");
699 std::pair<SENode*, SENode*> subscript_pair =
700 std::make_pair(source, destination);
701 const Loop* subscript_loop = GetLoopForSubscriptPair(subscript_pair);
702 // Build an SENode for distance.
703 SENode* destination_constant_term =
704 GetConstantTerm(subscript_loop, destination);
705 SENode* delta = scalar_evolution_.SimplifyExpression(
706 scalar_evolution_.CreateSubtraction(source, destination_constant_term));
707
708 // Scalar evolution doesn't perform division, so we must fold to constants and
709 // do it manually.
710 int64_t distance = 0;
711 SEConstantNode* delta_constant = delta->AsSEConstantNode();
712 SEConstantNode* coefficient_constant = coefficient->AsSEConstantNode();
713 if (delta_constant && coefficient_constant) {
714 PrintDebug(
715 "WeakZeroSourceSIVTest folding delta and coefficient to constants.");
716 int64_t delta_value = delta_constant->FoldToSingleValue();
717 int64_t coefficient_value = coefficient_constant->FoldToSingleValue();
718 // Check if the distance is not integral.
719 if (delta_value % coefficient_value != 0) {
720 PrintDebug(
721 "WeakZeroSourceSIVTest proved independence through distance not "
722 "being an integer.");
723 distance_entry->dependence_information =
724 DistanceEntry::DependenceInformation::DIRECTION;
725 distance_entry->direction = DistanceEntry::Directions::NONE;
726 return true;
727 } else {
728 distance = delta_value / coefficient_value;
729 PrintDebug(
730 "WeakZeroSourceSIVTest calculated distance with the following "
731 "values\n"
732 "\tdelta value: " +
733 ToString(delta_value) +
734 "\n\tcoefficient value: " + ToString(coefficient_value) +
735 "\n\tdistance: " + ToString(distance) + "\n");
736 }
737 } else {
738 PrintDebug(
739 "WeakZeroSourceSIVTest was unable to fold delta and coefficient to "
740 "constants.");
741 }
742
743 // If we can prove the distance is outside the bounds we prove independence.
744 SEConstantNode* lower_bound =
745 GetLowerBound(subscript_loop)->AsSEConstantNode();
746 SEConstantNode* upper_bound =
747 GetUpperBound(subscript_loop)->AsSEConstantNode();
748 if (lower_bound && upper_bound) {
749 PrintDebug("WeakZeroSourceSIVTest found bounds as SEConstantNodes.");
750 int64_t lower_bound_value = lower_bound->FoldToSingleValue();
751 int64_t upper_bound_value = upper_bound->FoldToSingleValue();
752 if (!IsWithinBounds(llabs(distance), lower_bound_value,
753 upper_bound_value)) {
754 PrintDebug(
755 "WeakZeroSourceSIVTest proved independence through distance escaping "
756 "the loop bounds.");
757 PrintDebug(
758 "Bound values were as follow\n"
759 "\tlower bound value: " +
760 ToString(lower_bound_value) +
761 "\n\tupper bound value: " + ToString(upper_bound_value) +
762 "\n\tdistance value: " + ToString(distance) + "\n");
763 distance_entry->dependence_information =
764 DistanceEntry::DependenceInformation::DISTANCE;
765 distance_entry->direction = DistanceEntry::Directions::NONE;
766 distance_entry->distance = distance;
767 return true;
768 }
769 } else {
770 PrintDebug(
771 "WeakZeroSourceSIVTest was unable to find lower and upper bound as "
772 "SEConstantNodes.");
773 }
774
775 // Now we want to see if we can detect to peel the first or last iterations.
776
777 // We get the FirstTripValue as GetFirstTripInductionNode() +
778 // GetConstantTerm(destination)
779 SENode* first_trip_SENode =
780 scalar_evolution_.SimplifyExpression(scalar_evolution_.CreateAddNode(
781 GetFirstTripInductionNode(subscript_loop),
782 GetConstantTerm(subscript_loop, destination)));
783
784 // If source == FirstTripValue, peel_first.
785 if (first_trip_SENode) {
786 PrintDebug("WeakZeroSourceSIVTest built first_trip_SENode.");
787 if (first_trip_SENode->AsSEConstantNode()) {
788 PrintDebug(
789 "WeakZeroSourceSIVTest has found first_trip_SENode as an "
790 "SEConstantNode with value: " +
791 ToString(first_trip_SENode->AsSEConstantNode()->FoldToSingleValue()) +
792 "\n");
793 }
794 if (source == first_trip_SENode) {
795 // We have found that peeling the first iteration will break dependency.
796 PrintDebug(
797 "WeakZeroSourceSIVTest has found peeling first iteration will break "
798 "dependency");
799 distance_entry->dependence_information =
800 DistanceEntry::DependenceInformation::PEEL;
801 distance_entry->peel_first = true;
802 return false;
803 }
804 } else {
805 PrintDebug("WeakZeroSourceSIVTest was unable to build first_trip_SENode");
806 }
807
808 // We get the LastTripValue as GetFinalTripInductionNode(coefficient) +
809 // GetConstantTerm(destination)
810 SENode* final_trip_SENode =
811 scalar_evolution_.SimplifyExpression(scalar_evolution_.CreateAddNode(
812 GetFinalTripInductionNode(subscript_loop, coefficient),
813 GetConstantTerm(subscript_loop, destination)));
814
815 // If source == LastTripValue, peel_last.
816 if (final_trip_SENode) {
817 PrintDebug("WeakZeroSourceSIVTest built final_trip_SENode.");
818 if (first_trip_SENode->AsSEConstantNode()) {
819 PrintDebug(
820 "WeakZeroSourceSIVTest has found final_trip_SENode as an "
821 "SEConstantNode with value: " +
822 ToString(final_trip_SENode->AsSEConstantNode()->FoldToSingleValue()) +
823 "\n");
824 }
825 if (source == final_trip_SENode) {
826 // We have found that peeling the last iteration will break dependency.
827 PrintDebug(
828 "WeakZeroSourceSIVTest has found peeling final iteration will break "
829 "dependency");
830 distance_entry->dependence_information =
831 DistanceEntry::DependenceInformation::PEEL;
832 distance_entry->peel_last = true;
833 return false;
834 }
835 } else {
836 PrintDebug("WeakZeroSourceSIVTest was unable to build final_trip_SENode");
837 }
838
839 // We were unable to prove independence or discern any additional information.
840 // Must assume <=> direction.
841 PrintDebug(
842 "WeakZeroSourceSIVTest was unable to determine any dependence "
843 "information.");
844 distance_entry->direction = DistanceEntry::Directions::ALL;
845 return false;
846 }
847
WeakZeroDestinationSIVTest(SERecurrentNode * source,SENode * destination,SENode * coefficient,DistanceEntry * distance_entry)848 bool LoopDependenceAnalysis::WeakZeroDestinationSIVTest(
849 SERecurrentNode* source, SENode* destination, SENode* coefficient,
850 DistanceEntry* distance_entry) {
851 PrintDebug("Performing WeakZeroDestinationSIVTest.");
852 // Build an SENode for distance.
853 std::pair<SENode*, SENode*> subscript_pair =
854 std::make_pair(source, destination);
855 const Loop* subscript_loop = GetLoopForSubscriptPair(subscript_pair);
856 SENode* source_constant_term = GetConstantTerm(subscript_loop, source);
857 SENode* delta = scalar_evolution_.SimplifyExpression(
858 scalar_evolution_.CreateSubtraction(destination, source_constant_term));
859
860 // Scalar evolution doesn't perform division, so we must fold to constants and
861 // do it manually.
862 int64_t distance = 0;
863 SEConstantNode* delta_constant = delta->AsSEConstantNode();
864 SEConstantNode* coefficient_constant = coefficient->AsSEConstantNode();
865 if (delta_constant && coefficient_constant) {
866 PrintDebug(
867 "WeakZeroDestinationSIVTest folding delta and coefficient to "
868 "constants.");
869 int64_t delta_value = delta_constant->FoldToSingleValue();
870 int64_t coefficient_value = coefficient_constant->FoldToSingleValue();
871 // Check if the distance is not integral.
872 if (delta_value % coefficient_value != 0) {
873 PrintDebug(
874 "WeakZeroDestinationSIVTest proved independence through distance not "
875 "being an integer.");
876 distance_entry->dependence_information =
877 DistanceEntry::DependenceInformation::DIRECTION;
878 distance_entry->direction = DistanceEntry::Directions::NONE;
879 return true;
880 } else {
881 distance = delta_value / coefficient_value;
882 PrintDebug(
883 "WeakZeroDestinationSIVTest calculated distance with the following "
884 "values\n"
885 "\tdelta value: " +
886 ToString(delta_value) +
887 "\n\tcoefficient value: " + ToString(coefficient_value) +
888 "\n\tdistance: " + ToString(distance) + "\n");
889 }
890 } else {
891 PrintDebug(
892 "WeakZeroDestinationSIVTest was unable to fold delta and coefficient "
893 "to constants.");
894 }
895
896 // If we can prove the distance is outside the bounds we prove independence.
897 SEConstantNode* lower_bound =
898 GetLowerBound(subscript_loop)->AsSEConstantNode();
899 SEConstantNode* upper_bound =
900 GetUpperBound(subscript_loop)->AsSEConstantNode();
901 if (lower_bound && upper_bound) {
902 PrintDebug("WeakZeroDestinationSIVTest found bounds as SEConstantNodes.");
903 int64_t lower_bound_value = lower_bound->FoldToSingleValue();
904 int64_t upper_bound_value = upper_bound->FoldToSingleValue();
905 if (!IsWithinBounds(llabs(distance), lower_bound_value,
906 upper_bound_value)) {
907 PrintDebug(
908 "WeakZeroDestinationSIVTest proved independence through distance "
909 "escaping the loop bounds.");
910 PrintDebug(
911 "Bound values were as follows\n"
912 "\tlower bound value: " +
913 ToString(lower_bound_value) +
914 "\n\tupper bound value: " + ToString(upper_bound_value) +
915 "\n\tdistance value: " + ToString(distance));
916 distance_entry->dependence_information =
917 DistanceEntry::DependenceInformation::DISTANCE;
918 distance_entry->direction = DistanceEntry::Directions::NONE;
919 distance_entry->distance = distance;
920 return true;
921 }
922 } else {
923 PrintDebug(
924 "WeakZeroDestinationSIVTest was unable to find lower and upper bound "
925 "as SEConstantNodes.");
926 }
927
928 // Now we want to see if we can detect to peel the first or last iterations.
929
930 // We get the FirstTripValue as GetFirstTripInductionNode() +
931 // GetConstantTerm(source)
932 SENode* first_trip_SENode = scalar_evolution_.SimplifyExpression(
933 scalar_evolution_.CreateAddNode(GetFirstTripInductionNode(subscript_loop),
934 GetConstantTerm(subscript_loop, source)));
935
936 // If destination == FirstTripValue, peel_first.
937 if (first_trip_SENode) {
938 PrintDebug("WeakZeroDestinationSIVTest built first_trip_SENode.");
939 if (first_trip_SENode->AsSEConstantNode()) {
940 PrintDebug(
941 "WeakZeroDestinationSIVTest has found first_trip_SENode as an "
942 "SEConstantNode with value: " +
943 ToString(first_trip_SENode->AsSEConstantNode()->FoldToSingleValue()) +
944 "\n");
945 }
946 if (destination == first_trip_SENode) {
947 // We have found that peeling the first iteration will break dependency.
948 PrintDebug(
949 "WeakZeroDestinationSIVTest has found peeling first iteration will "
950 "break dependency");
951 distance_entry->dependence_information =
952 DistanceEntry::DependenceInformation::PEEL;
953 distance_entry->peel_first = true;
954 return false;
955 }
956 } else {
957 PrintDebug(
958 "WeakZeroDestinationSIVTest was unable to build first_trip_SENode");
959 }
960
961 // We get the LastTripValue as GetFinalTripInductionNode(coefficient) +
962 // GetConstantTerm(source)
963 SENode* final_trip_SENode =
964 scalar_evolution_.SimplifyExpression(scalar_evolution_.CreateAddNode(
965 GetFinalTripInductionNode(subscript_loop, coefficient),
966 GetConstantTerm(subscript_loop, source)));
967
968 // If destination == LastTripValue, peel_last.
969 if (final_trip_SENode) {
970 PrintDebug("WeakZeroDestinationSIVTest built final_trip_SENode.");
971 if (final_trip_SENode->AsSEConstantNode()) {
972 PrintDebug(
973 "WeakZeroDestinationSIVTest has found final_trip_SENode as an "
974 "SEConstantNode with value: " +
975 ToString(final_trip_SENode->AsSEConstantNode()->FoldToSingleValue()) +
976 "\n");
977 }
978 if (destination == final_trip_SENode) {
979 // We have found that peeling the last iteration will break dependency.
980 PrintDebug(
981 "WeakZeroDestinationSIVTest has found peeling final iteration will "
982 "break dependency");
983 distance_entry->dependence_information =
984 DistanceEntry::DependenceInformation::PEEL;
985 distance_entry->peel_last = true;
986 return false;
987 }
988 } else {
989 PrintDebug(
990 "WeakZeroDestinationSIVTest was unable to build final_trip_SENode");
991 }
992
993 // We were unable to prove independence or discern any additional information.
994 // Must assume <=> direction.
995 PrintDebug(
996 "WeakZeroDestinationSIVTest was unable to determine any dependence "
997 "information.");
998 distance_entry->direction = DistanceEntry::Directions::ALL;
999 return false;
1000 }
1001
WeakCrossingSIVTest(SENode * source,SENode * destination,SENode * coefficient,DistanceEntry * distance_entry)1002 bool LoopDependenceAnalysis::WeakCrossingSIVTest(
1003 SENode* source, SENode* destination, SENode* coefficient,
1004 DistanceEntry* distance_entry) {
1005 PrintDebug("Performing WeakCrossingSIVTest.");
1006 // We currently can't handle symbolic WeakCrossingSIVTests. If either source
1007 // or destination are not SERecurrentNodes we must exit.
1008 if (!source->AsSERecurrentNode() || !destination->AsSERecurrentNode()) {
1009 PrintDebug(
1010 "WeakCrossingSIVTest found source or destination != SERecurrentNode. "
1011 "Exiting");
1012 distance_entry->direction = DistanceEntry::Directions::ALL;
1013 return false;
1014 }
1015
1016 // Build an SENode for distance.
1017 SENode* offset_delta =
1018 scalar_evolution_.SimplifyExpression(scalar_evolution_.CreateSubtraction(
1019 destination->AsSERecurrentNode()->GetOffset(),
1020 source->AsSERecurrentNode()->GetOffset()));
1021
1022 // Scalar evolution doesn't perform division, so we must fold to constants and
1023 // do it manually.
1024 int64_t distance = 0;
1025 SEConstantNode* delta_constant = offset_delta->AsSEConstantNode();
1026 SEConstantNode* coefficient_constant = coefficient->AsSEConstantNode();
1027 if (delta_constant && coefficient_constant) {
1028 PrintDebug(
1029 "WeakCrossingSIVTest folding offset_delta and coefficient to "
1030 "constants.");
1031 int64_t delta_value = delta_constant->FoldToSingleValue();
1032 int64_t coefficient_value = coefficient_constant->FoldToSingleValue();
1033 // Check if the distance is not integral or if it has a non-integral part
1034 // equal to 1/2.
1035 if (delta_value % (2 * coefficient_value) != 0 &&
1036 static_cast<float>(delta_value % (2 * coefficient_value)) /
1037 static_cast<float>(2 * coefficient_value) !=
1038 0.5) {
1039 PrintDebug(
1040 "WeakCrossingSIVTest proved independence through distance escaping "
1041 "the loop bounds.");
1042 distance_entry->dependence_information =
1043 DistanceEntry::DependenceInformation::DIRECTION;
1044 distance_entry->direction = DistanceEntry::Directions::NONE;
1045 return true;
1046 } else {
1047 distance = delta_value / (2 * coefficient_value);
1048 }
1049
1050 if (distance == 0) {
1051 PrintDebug("WeakCrossingSIVTest found EQ dependence.");
1052 distance_entry->dependence_information =
1053 DistanceEntry::DependenceInformation::DISTANCE;
1054 distance_entry->direction = DistanceEntry::Directions::EQ;
1055 distance_entry->distance = 0;
1056 return false;
1057 }
1058 } else {
1059 PrintDebug(
1060 "WeakCrossingSIVTest was unable to fold offset_delta and coefficient "
1061 "to constants.");
1062 }
1063
1064 // We were unable to prove independence or discern any additional information.
1065 // Must assume <=> direction.
1066 PrintDebug(
1067 "WeakCrossingSIVTest was unable to determine any dependence "
1068 "information.");
1069 distance_entry->direction = DistanceEntry::Directions::ALL;
1070 return false;
1071 }
1072
1073 // Perform the GCD test if both, the source and the destination nodes, are in
1074 // the form a0*i0 + a1*i1 + ... an*in + c.
GCDMIVTest(const std::pair<SENode *,SENode * > & subscript_pair)1075 bool LoopDependenceAnalysis::GCDMIVTest(
1076 const std::pair<SENode*, SENode*>& subscript_pair) {
1077 auto source = std::get<0>(subscript_pair);
1078 auto destination = std::get<1>(subscript_pair);
1079
1080 // Bail out if source/destination is in an unexpected form.
1081 if (!IsInCorrectFormForGCDTest(source) ||
1082 !IsInCorrectFormForGCDTest(destination)) {
1083 return false;
1084 }
1085
1086 auto source_recurrences = GetAllTopLevelRecurrences(source);
1087 auto dest_recurrences = GetAllTopLevelRecurrences(destination);
1088
1089 // Bail out if all offsets and coefficients aren't constant.
1090 if (!AreOffsetsAndCoefficientsConstant(source_recurrences) ||
1091 !AreOffsetsAndCoefficientsConstant(dest_recurrences)) {
1092 return false;
1093 }
1094
1095 // Calculate the GCD of all coefficients.
1096 auto source_constants = GetAllTopLevelConstants(source);
1097 int64_t source_constant =
1098 CalculateConstantTerm(source_recurrences, source_constants);
1099
1100 auto dest_constants = GetAllTopLevelConstants(destination);
1101 int64_t destination_constant =
1102 CalculateConstantTerm(dest_recurrences, dest_constants);
1103
1104 int64_t delta = std::abs(source_constant - destination_constant);
1105
1106 int64_t running_gcd = 0;
1107
1108 running_gcd = CalculateGCDFromCoefficients(source_recurrences, running_gcd);
1109 running_gcd = CalculateGCDFromCoefficients(dest_recurrences, running_gcd);
1110
1111 return delta % running_gcd != 0;
1112 }
1113
1114 using PartitionedSubscripts =
1115 std::vector<std::set<std::pair<Instruction*, Instruction*>>>;
PartitionSubscripts(const std::vector<Instruction * > & source_subscripts,const std::vector<Instruction * > & destination_subscripts)1116 PartitionedSubscripts LoopDependenceAnalysis::PartitionSubscripts(
1117 const std::vector<Instruction*>& source_subscripts,
1118 const std::vector<Instruction*>& destination_subscripts) {
1119 PartitionedSubscripts partitions{};
1120
1121 auto num_subscripts = source_subscripts.size();
1122
1123 // Create initial partitions with one subscript pair per partition.
1124 for (size_t i = 0; i < num_subscripts; ++i) {
1125 partitions.push_back({{source_subscripts[i], destination_subscripts[i]}});
1126 }
1127
1128 // Iterate over the loops to create all partitions
1129 for (auto loop : loops_) {
1130 int64_t k = -1;
1131
1132 for (size_t j = 0; j < partitions.size(); ++j) {
1133 auto& current_partition = partitions[j];
1134
1135 // Does |loop| appear in |current_partition|
1136 auto it = std::find_if(
1137 current_partition.begin(), current_partition.end(),
1138 [loop,
1139 this](const std::pair<Instruction*, Instruction*>& elem) -> bool {
1140 auto source_recurrences =
1141 scalar_evolution_.AnalyzeInstruction(std::get<0>(elem))
1142 ->CollectRecurrentNodes();
1143 auto destination_recurrences =
1144 scalar_evolution_.AnalyzeInstruction(std::get<1>(elem))
1145 ->CollectRecurrentNodes();
1146
1147 source_recurrences.insert(source_recurrences.end(),
1148 destination_recurrences.begin(),
1149 destination_recurrences.end());
1150
1151 auto loops_in_pair = CollectLoops(source_recurrences);
1152 auto end_it = loops_in_pair.end();
1153
1154 return std::find(loops_in_pair.begin(), end_it, loop) != end_it;
1155 });
1156
1157 auto has_loop = it != current_partition.end();
1158
1159 if (has_loop) {
1160 if (k == -1) {
1161 k = j;
1162 } else {
1163 // Add |partitions[j]| to |partitions[k]| and discard |partitions[j]|
1164 partitions[static_cast<size_t>(k)].insert(current_partition.begin(),
1165 current_partition.end());
1166 current_partition.clear();
1167 }
1168 }
1169 }
1170 }
1171
1172 // Remove empty (discarded) partitions
1173 partitions.erase(
1174 std::remove_if(
1175 partitions.begin(), partitions.end(),
1176 [](const std::set<std::pair<Instruction*, Instruction*>>& partition) {
1177 return partition.empty();
1178 }),
1179 partitions.end());
1180
1181 return partitions;
1182 }
1183
IntersectConstraints(Constraint * constraint_0,Constraint * constraint_1,const SENode * lower_bound,const SENode * upper_bound)1184 Constraint* LoopDependenceAnalysis::IntersectConstraints(
1185 Constraint* constraint_0, Constraint* constraint_1,
1186 const SENode* lower_bound, const SENode* upper_bound) {
1187 if (constraint_0->AsDependenceNone()) {
1188 return constraint_1;
1189 } else if (constraint_1->AsDependenceNone()) {
1190 return constraint_0;
1191 }
1192
1193 // Both constraints are distances. Either the same distance or independent.
1194 if (constraint_0->AsDependenceDistance() &&
1195 constraint_1->AsDependenceDistance()) {
1196 auto dist_0 = constraint_0->AsDependenceDistance();
1197 auto dist_1 = constraint_1->AsDependenceDistance();
1198
1199 if (*dist_0->GetDistance() == *dist_1->GetDistance()) {
1200 return constraint_0;
1201 } else {
1202 return make_constraint<DependenceEmpty>();
1203 }
1204 }
1205
1206 // Both constraints are points. Either the same point or independent.
1207 if (constraint_0->AsDependencePoint() && constraint_1->AsDependencePoint()) {
1208 auto point_0 = constraint_0->AsDependencePoint();
1209 auto point_1 = constraint_1->AsDependencePoint();
1210
1211 if (*point_0->GetSource() == *point_1->GetSource() &&
1212 *point_0->GetDestination() == *point_1->GetDestination()) {
1213 return constraint_0;
1214 } else {
1215 return make_constraint<DependenceEmpty>();
1216 }
1217 }
1218
1219 // Both constraints are lines/distances.
1220 if ((constraint_0->AsDependenceDistance() ||
1221 constraint_0->AsDependenceLine()) &&
1222 (constraint_1->AsDependenceDistance() ||
1223 constraint_1->AsDependenceLine())) {
1224 auto is_distance_0 = constraint_0->AsDependenceDistance() != nullptr;
1225 auto is_distance_1 = constraint_1->AsDependenceDistance() != nullptr;
1226
1227 auto a0 = is_distance_0 ? scalar_evolution_.CreateConstant(1)
1228 : constraint_0->AsDependenceLine()->GetA();
1229 auto b0 = is_distance_0 ? scalar_evolution_.CreateConstant(-1)
1230 : constraint_0->AsDependenceLine()->GetB();
1231 auto c0 =
1232 is_distance_0
1233 ? scalar_evolution_.SimplifyExpression(
1234 scalar_evolution_.CreateNegation(
1235 constraint_0->AsDependenceDistance()->GetDistance()))
1236 : constraint_0->AsDependenceLine()->GetC();
1237
1238 auto a1 = is_distance_1 ? scalar_evolution_.CreateConstant(1)
1239 : constraint_1->AsDependenceLine()->GetA();
1240 auto b1 = is_distance_1 ? scalar_evolution_.CreateConstant(-1)
1241 : constraint_1->AsDependenceLine()->GetB();
1242 auto c1 =
1243 is_distance_1
1244 ? scalar_evolution_.SimplifyExpression(
1245 scalar_evolution_.CreateNegation(
1246 constraint_1->AsDependenceDistance()->GetDistance()))
1247 : constraint_1->AsDependenceLine()->GetC();
1248
1249 if (a0->AsSEConstantNode() && b0->AsSEConstantNode() &&
1250 c0->AsSEConstantNode() && a1->AsSEConstantNode() &&
1251 b1->AsSEConstantNode() && c1->AsSEConstantNode()) {
1252 auto constant_a0 = a0->AsSEConstantNode()->FoldToSingleValue();
1253 auto constant_b0 = b0->AsSEConstantNode()->FoldToSingleValue();
1254 auto constant_c0 = c0->AsSEConstantNode()->FoldToSingleValue();
1255
1256 auto constant_a1 = a1->AsSEConstantNode()->FoldToSingleValue();
1257 auto constant_b1 = b1->AsSEConstantNode()->FoldToSingleValue();
1258 auto constant_c1 = c1->AsSEConstantNode()->FoldToSingleValue();
1259
1260 // a & b can't both be zero, otherwise it wouldn't be line.
1261 if (NormalizeAndCompareFractions(constant_a0, constant_b0, constant_a1,
1262 constant_b1)) {
1263 // Slopes are equal, either parallel lines or the same line.
1264
1265 if (constant_b0 == 0 && constant_b1 == 0) {
1266 if (NormalizeAndCompareFractions(constant_c0, constant_a0,
1267 constant_c1, constant_a1)) {
1268 return constraint_0;
1269 }
1270
1271 return make_constraint<DependenceEmpty>();
1272 } else if (NormalizeAndCompareFractions(constant_c0, constant_b0,
1273 constant_c1, constant_b1)) {
1274 // Same line.
1275 return constraint_0;
1276 } else {
1277 // Parallel lines can't intersect, report independence.
1278 return make_constraint<DependenceEmpty>();
1279 }
1280
1281 } else {
1282 // Lines are not parallel, therefore, they must intersect.
1283
1284 // Calculate intersection.
1285 if (upper_bound->AsSEConstantNode() &&
1286 lower_bound->AsSEConstantNode()) {
1287 auto constant_lower_bound =
1288 lower_bound->AsSEConstantNode()->FoldToSingleValue();
1289 auto constant_upper_bound =
1290 upper_bound->AsSEConstantNode()->FoldToSingleValue();
1291
1292 auto up = constant_b1 * constant_c0 - constant_b0 * constant_c1;
1293 // Both b or both a can't be 0, so down is never 0
1294 // otherwise would have entered the parallel line section.
1295 auto down = constant_b1 * constant_a0 - constant_b0 * constant_a1;
1296
1297 auto x_coord = up / down;
1298
1299 int64_t y_coord = 0;
1300 int64_t arg1 = 0;
1301 int64_t const_b_to_use = 0;
1302
1303 if (constant_b1 != 0) {
1304 arg1 = constant_c1 - constant_a1 * x_coord;
1305 y_coord = arg1 / constant_b1;
1306 const_b_to_use = constant_b1;
1307 } else if (constant_b0 != 0) {
1308 arg1 = constant_c0 - constant_a0 * x_coord;
1309 y_coord = arg1 / constant_b0;
1310 const_b_to_use = constant_b0;
1311 }
1312
1313 if (up % down == 0 &&
1314 arg1 % const_b_to_use == 0 && // Coordinates are integers.
1315 constant_lower_bound <=
1316 x_coord && // x_coord is within loop bounds.
1317 x_coord <= constant_upper_bound &&
1318 constant_lower_bound <=
1319 y_coord && // y_coord is within loop bounds.
1320 y_coord <= constant_upper_bound) {
1321 // Lines intersect at integer coordinates.
1322 return make_constraint<DependencePoint>(
1323 scalar_evolution_.CreateConstant(x_coord),
1324 scalar_evolution_.CreateConstant(y_coord),
1325 constraint_0->GetLoop());
1326
1327 } else {
1328 return make_constraint<DependenceEmpty>();
1329 }
1330
1331 } else {
1332 // Not constants, bail out.
1333 return make_constraint<DependenceNone>();
1334 }
1335 }
1336
1337 } else {
1338 // Not constants, bail out.
1339 return make_constraint<DependenceNone>();
1340 }
1341 }
1342
1343 // One constraint is a line/distance and the other is a point.
1344 if ((constraint_0->AsDependencePoint() &&
1345 (constraint_1->AsDependenceLine() ||
1346 constraint_1->AsDependenceDistance())) ||
1347 (constraint_1->AsDependencePoint() &&
1348 (constraint_0->AsDependenceLine() ||
1349 constraint_0->AsDependenceDistance()))) {
1350 auto point_0 = constraint_0->AsDependencePoint() != nullptr;
1351
1352 auto point = point_0 ? constraint_0->AsDependencePoint()
1353 : constraint_1->AsDependencePoint();
1354
1355 auto line_or_distance = point_0 ? constraint_1 : constraint_0;
1356
1357 auto is_distance = line_or_distance->AsDependenceDistance() != nullptr;
1358
1359 auto a = is_distance ? scalar_evolution_.CreateConstant(1)
1360 : line_or_distance->AsDependenceLine()->GetA();
1361 auto b = is_distance ? scalar_evolution_.CreateConstant(-1)
1362 : line_or_distance->AsDependenceLine()->GetB();
1363 auto c =
1364 is_distance
1365 ? scalar_evolution_.SimplifyExpression(
1366 scalar_evolution_.CreateNegation(
1367 line_or_distance->AsDependenceDistance()->GetDistance()))
1368 : line_or_distance->AsDependenceLine()->GetC();
1369
1370 auto x = point->GetSource();
1371 auto y = point->GetDestination();
1372
1373 if (a->AsSEConstantNode() && b->AsSEConstantNode() &&
1374 c->AsSEConstantNode() && x->AsSEConstantNode() &&
1375 y->AsSEConstantNode()) {
1376 auto constant_a = a->AsSEConstantNode()->FoldToSingleValue();
1377 auto constant_b = b->AsSEConstantNode()->FoldToSingleValue();
1378 auto constant_c = c->AsSEConstantNode()->FoldToSingleValue();
1379
1380 auto constant_x = x->AsSEConstantNode()->FoldToSingleValue();
1381 auto constant_y = y->AsSEConstantNode()->FoldToSingleValue();
1382
1383 auto left_hand_side = constant_a * constant_x + constant_b * constant_y;
1384
1385 if (left_hand_side == constant_c) {
1386 // Point is on line, return point
1387 return point_0 ? constraint_0 : constraint_1;
1388 } else {
1389 // Point not on line, report independence (empty constraint).
1390 return make_constraint<DependenceEmpty>();
1391 }
1392
1393 } else {
1394 // Not constants, bail out.
1395 return make_constraint<DependenceNone>();
1396 }
1397 }
1398
1399 return nullptr;
1400 }
1401
1402 // Propagate constraints function as described in section 5 of Practical
1403 // Dependence Testing, Goff, Kennedy, Tseng, 1991.
PropagateConstraints(const SubscriptPair & subscript_pair,const std::vector<Constraint * > & constraints)1404 SubscriptPair LoopDependenceAnalysis::PropagateConstraints(
1405 const SubscriptPair& subscript_pair,
1406 const std::vector<Constraint*>& constraints) {
1407 SENode* new_first = subscript_pair.first;
1408 SENode* new_second = subscript_pair.second;
1409
1410 for (auto& constraint : constraints) {
1411 // In the paper this is a[k]. We're extracting the coefficient ('a') of a
1412 // recurrent expression with respect to the loop 'k'.
1413 SENode* coefficient_of_recurrent =
1414 scalar_evolution_.GetCoefficientFromRecurrentTerm(
1415 new_first, constraint->GetLoop());
1416
1417 // In the paper this is a'[k].
1418 SENode* coefficient_of_recurrent_prime =
1419 scalar_evolution_.GetCoefficientFromRecurrentTerm(
1420 new_second, constraint->GetLoop());
1421
1422 if (constraint->GetType() == Constraint::Distance) {
1423 DependenceDistance* as_distance = constraint->AsDependenceDistance();
1424
1425 // In the paper this is a[k]*d
1426 SENode* rhs = scalar_evolution_.CreateMultiplyNode(
1427 coefficient_of_recurrent, as_distance->GetDistance());
1428
1429 // In the paper this is a[k] <- 0
1430 SENode* zeroed_coefficient =
1431 scalar_evolution_.BuildGraphWithoutRecurrentTerm(
1432 new_first, constraint->GetLoop());
1433
1434 // In the paper this is e <- e - a[k]*d.
1435 new_first = scalar_evolution_.CreateSubtraction(zeroed_coefficient, rhs);
1436 new_first = scalar_evolution_.SimplifyExpression(new_first);
1437
1438 // In the paper this is a'[k] - a[k].
1439 SENode* new_child = scalar_evolution_.SimplifyExpression(
1440 scalar_evolution_.CreateSubtraction(coefficient_of_recurrent_prime,
1441 coefficient_of_recurrent));
1442
1443 // In the paper this is a'[k]'i[k].
1444 SERecurrentNode* prime_recurrent =
1445 scalar_evolution_.GetRecurrentTerm(new_second, constraint->GetLoop());
1446
1447 if (!prime_recurrent) continue;
1448
1449 // As we hash the nodes we need to create a new node when we update a
1450 // child.
1451 SENode* new_recurrent = scalar_evolution_.CreateRecurrentExpression(
1452 constraint->GetLoop(), prime_recurrent->GetOffset(), new_child);
1453 // In the paper this is a'[k] <- a'[k] - a[k].
1454 new_second = scalar_evolution_.UpdateChildNode(
1455 new_second, prime_recurrent, new_recurrent);
1456 }
1457 }
1458
1459 new_second = scalar_evolution_.SimplifyExpression(new_second);
1460 return std::make_pair(new_first, new_second);
1461 }
1462
DeltaTest(const std::vector<SubscriptPair> & coupled_subscripts,DistanceVector * dv_entry)1463 bool LoopDependenceAnalysis::DeltaTest(
1464 const std::vector<SubscriptPair>& coupled_subscripts,
1465 DistanceVector* dv_entry) {
1466 std::vector<Constraint*> constraints(loops_.size());
1467
1468 std::vector<bool> loop_appeared(loops_.size());
1469
1470 std::generate(std::begin(constraints), std::end(constraints),
1471 [this]() { return make_constraint<DependenceNone>(); });
1472
1473 // Separate SIV and MIV subscripts
1474 std::vector<SubscriptPair> siv_subscripts{};
1475 std::vector<SubscriptPair> miv_subscripts{};
1476
1477 for (const auto& subscript_pair : coupled_subscripts) {
1478 if (IsSIV(subscript_pair)) {
1479 siv_subscripts.push_back(subscript_pair);
1480 } else {
1481 miv_subscripts.push_back(subscript_pair);
1482 }
1483 }
1484
1485 // Delta Test
1486 while (!siv_subscripts.empty()) {
1487 std::vector<bool> results(siv_subscripts.size());
1488
1489 std::vector<DistanceVector> current_distances(
1490 siv_subscripts.size(), DistanceVector(loops_.size()));
1491
1492 // Apply SIV test to all SIV subscripts, report independence if any of them
1493 // is independent
1494 std::transform(
1495 std::begin(siv_subscripts), std::end(siv_subscripts),
1496 std::begin(current_distances), std::begin(results),
1497 [this](SubscriptPair& p, DistanceVector& d) { return SIVTest(p, &d); });
1498
1499 if (std::accumulate(std::begin(results), std::end(results), false,
1500 std::logical_or<bool>{})) {
1501 return true;
1502 }
1503
1504 // Derive new constraint vector.
1505 std::vector<std::pair<Constraint*, size_t>> all_new_constrants{};
1506
1507 for (size_t i = 0; i < siv_subscripts.size(); ++i) {
1508 auto loop = GetLoopForSubscriptPair(siv_subscripts[i]);
1509
1510 auto loop_id =
1511 std::distance(std::begin(loops_),
1512 std::find(std::begin(loops_), std::end(loops_), loop));
1513
1514 loop_appeared[loop_id] = true;
1515 auto distance_entry = current_distances[i].GetEntries()[loop_id];
1516
1517 if (distance_entry.dependence_information ==
1518 DistanceEntry::DependenceInformation::DISTANCE) {
1519 // Construct a DependenceDistance.
1520 auto node = scalar_evolution_.CreateConstant(distance_entry.distance);
1521
1522 all_new_constrants.push_back(
1523 {make_constraint<DependenceDistance>(node, loop), loop_id});
1524 } else {
1525 // Construct a DependenceLine.
1526 const auto& subscript_pair = siv_subscripts[i];
1527 SENode* source_node = std::get<0>(subscript_pair);
1528 SENode* destination_node = std::get<1>(subscript_pair);
1529
1530 int64_t source_induction_count = CountInductionVariables(source_node);
1531 int64_t destination_induction_count =
1532 CountInductionVariables(destination_node);
1533
1534 SENode* a = nullptr;
1535 SENode* b = nullptr;
1536 SENode* c = nullptr;
1537
1538 if (destination_induction_count != 0) {
1539 a = destination_node->AsSERecurrentNode()->GetCoefficient();
1540 c = scalar_evolution_.CreateNegation(
1541 destination_node->AsSERecurrentNode()->GetOffset());
1542 } else {
1543 a = scalar_evolution_.CreateConstant(0);
1544 c = scalar_evolution_.CreateNegation(destination_node);
1545 }
1546
1547 if (source_induction_count != 0) {
1548 b = scalar_evolution_.CreateNegation(
1549 source_node->AsSERecurrentNode()->GetCoefficient());
1550 c = scalar_evolution_.CreateAddNode(
1551 c, source_node->AsSERecurrentNode()->GetOffset());
1552 } else {
1553 b = scalar_evolution_.CreateConstant(0);
1554 c = scalar_evolution_.CreateAddNode(c, source_node);
1555 }
1556
1557 a = scalar_evolution_.SimplifyExpression(a);
1558 b = scalar_evolution_.SimplifyExpression(b);
1559 c = scalar_evolution_.SimplifyExpression(c);
1560
1561 all_new_constrants.push_back(
1562 {make_constraint<DependenceLine>(a, b, c, loop), loop_id});
1563 }
1564 }
1565
1566 // Calculate the intersection between the new and existing constraints.
1567 std::vector<Constraint*> intersection = constraints;
1568 for (const auto& constraint_to_intersect : all_new_constrants) {
1569 auto loop_id = std::get<1>(constraint_to_intersect);
1570 auto loop = loops_[loop_id];
1571 intersection[loop_id] = IntersectConstraints(
1572 intersection[loop_id], std::get<0>(constraint_to_intersect),
1573 GetLowerBound(loop), GetUpperBound(loop));
1574 }
1575
1576 // Report independence if an empty constraint (DependenceEmpty) is found.
1577 auto first_empty =
1578 std::find_if(std::begin(intersection), std::end(intersection),
1579 [](Constraint* constraint) {
1580 return constraint->AsDependenceEmpty() != nullptr;
1581 });
1582 if (first_empty != std::end(intersection)) {
1583 return true;
1584 }
1585 std::vector<SubscriptPair> new_siv_subscripts{};
1586 std::vector<SubscriptPair> new_miv_subscripts{};
1587
1588 auto equal =
1589 std::equal(std::begin(constraints), std::end(constraints),
1590 std::begin(intersection),
1591 [](Constraint* a, Constraint* b) { return *a == *b; });
1592
1593 // If any constraints have changed, propagate them into the rest of the
1594 // subscripts possibly creating new ZIV/SIV subscripts.
1595 if (!equal) {
1596 std::vector<SubscriptPair> new_subscripts(miv_subscripts.size());
1597
1598 // Propagate constraints into MIV subscripts
1599 std::transform(std::begin(miv_subscripts), std::end(miv_subscripts),
1600 std::begin(new_subscripts),
1601 [this, &intersection](SubscriptPair& subscript_pair) {
1602 return PropagateConstraints(subscript_pair,
1603 intersection);
1604 });
1605
1606 // If a ZIV subscript is returned, apply test, otherwise, update untested
1607 // subscripts.
1608 for (auto& subscript : new_subscripts) {
1609 if (IsZIV(subscript) && ZIVTest(subscript)) {
1610 return true;
1611 } else if (IsSIV(subscript)) {
1612 new_siv_subscripts.push_back(subscript);
1613 } else {
1614 new_miv_subscripts.push_back(subscript);
1615 }
1616 }
1617 }
1618
1619 // Set new constraints and subscripts to test.
1620 std::swap(siv_subscripts, new_siv_subscripts);
1621 std::swap(miv_subscripts, new_miv_subscripts);
1622 std::swap(constraints, intersection);
1623 }
1624
1625 // Create the dependence vector from the constraints.
1626 for (size_t i = 0; i < loops_.size(); ++i) {
1627 // Don't touch entries for loops that weren't tested.
1628 if (loop_appeared[i]) {
1629 auto current_constraint = constraints[i];
1630 auto& current_distance_entry = (*dv_entry).GetEntries()[i];
1631
1632 if (auto dependence_distance =
1633 current_constraint->AsDependenceDistance()) {
1634 if (auto constant_node =
1635 dependence_distance->GetDistance()->AsSEConstantNode()) {
1636 current_distance_entry.dependence_information =
1637 DistanceEntry::DependenceInformation::DISTANCE;
1638
1639 current_distance_entry.distance = constant_node->FoldToSingleValue();
1640 if (current_distance_entry.distance == 0) {
1641 current_distance_entry.direction = DistanceEntry::Directions::EQ;
1642 } else if (current_distance_entry.distance < 0) {
1643 current_distance_entry.direction = DistanceEntry::Directions::GT;
1644 } else {
1645 current_distance_entry.direction = DistanceEntry::Directions::LT;
1646 }
1647 }
1648 } else if (auto dependence_point =
1649 current_constraint->AsDependencePoint()) {
1650 auto source = dependence_point->GetSource();
1651 auto destination = dependence_point->GetDestination();
1652
1653 if (source->AsSEConstantNode() && destination->AsSEConstantNode()) {
1654 current_distance_entry = DistanceEntry(
1655 source->AsSEConstantNode()->FoldToSingleValue(),
1656 destination->AsSEConstantNode()->FoldToSingleValue());
1657 }
1658 }
1659 }
1660 }
1661
1662 // Test any remaining MIV subscripts and report independence if found.
1663 std::vector<bool> results(miv_subscripts.size());
1664
1665 std::transform(std::begin(miv_subscripts), std::end(miv_subscripts),
1666 std::begin(results),
1667 [this](const SubscriptPair& p) { return GCDMIVTest(p); });
1668
1669 return std::accumulate(std::begin(results), std::end(results), false,
1670 std::logical_or<bool>{});
1671 }
1672
1673 } // namespace opt
1674 } // namespace spvtools
1675