• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright (c) 2018 Google LLC.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //     http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "source/opt/loop_dependence.h"
16 
17 #include <functional>
18 #include <numeric>
19 #include <string>
20 #include <utility>
21 #include <vector>
22 
23 #include "source/opt/instruction.h"
24 #include "source/opt/scalar_analysis_nodes.h"
25 
26 namespace spvtools {
27 namespace opt {
28 
29 using SubscriptPair = std::pair<SENode*, SENode*>;
30 
31 namespace {
32 
33 // Calculate the greatest common divisor of a & b using Stein's algorithm.
34 // https://en.wikipedia.org/wiki/Binary_GCD_algorithm
GreatestCommonDivisor(int64_t a,int64_t b)35 int64_t GreatestCommonDivisor(int64_t a, int64_t b) {
36   // Simple cases
37   if (a == b) {
38     return a;
39   } else if (a == 0) {
40     return b;
41   } else if (b == 0) {
42     return a;
43   }
44 
45   // Both even
46   if (a % 2 == 0 && b % 2 == 0) {
47     return 2 * GreatestCommonDivisor(a / 2, b / 2);
48   }
49 
50   // Even a, odd b
51   if (a % 2 == 0 && b % 2 == 1) {
52     return GreatestCommonDivisor(a / 2, b);
53   }
54 
55   // Odd a, even b
56   if (a % 2 == 1 && b % 2 == 0) {
57     return GreatestCommonDivisor(a, b / 2);
58   }
59 
60   // Both odd, reduce the larger argument
61   if (a > b) {
62     return GreatestCommonDivisor((a - b) / 2, b);
63   } else {
64     return GreatestCommonDivisor((b - a) / 2, a);
65   }
66 }
67 
68 // Check if node is affine, ie in the form: a0*i0 + a1*i1 + ... an*in + c
69 // and contains only the following types of nodes: SERecurrentNode, SEAddNode
70 // and SEConstantNode
IsInCorrectFormForGCDTest(SENode * node)71 bool IsInCorrectFormForGCDTest(SENode* node) {
72   bool children_ok = true;
73 
74   if (auto add_node = node->AsSEAddNode()) {
75     for (auto child : add_node->GetChildren()) {
76       children_ok &= IsInCorrectFormForGCDTest(child);
77     }
78   }
79 
80   bool this_ok = node->AsSERecurrentNode() || node->AsSEAddNode() ||
81                  node->AsSEConstantNode();
82 
83   return children_ok && this_ok;
84 }
85 
86 // If |node| is an SERecurrentNode then returns |node| or if |node| is an
87 // SEAddNode returns a vector of SERecurrentNode that are its children.
GetAllTopLevelRecurrences(SENode * node)88 std::vector<SERecurrentNode*> GetAllTopLevelRecurrences(SENode* node) {
89   auto nodes = std::vector<SERecurrentNode*>{};
90   if (auto recurrent_node = node->AsSERecurrentNode()) {
91     nodes.push_back(recurrent_node);
92   }
93 
94   if (auto add_node = node->AsSEAddNode()) {
95     for (auto child : add_node->GetChildren()) {
96       auto child_nodes = GetAllTopLevelRecurrences(child);
97       nodes.insert(nodes.end(), child_nodes.begin(), child_nodes.end());
98     }
99   }
100 
101   return nodes;
102 }
103 
104 // If |node| is an SEConstantNode then returns |node| or if |node| is an
105 // SEAddNode returns a vector of SEConstantNode that are its children.
GetAllTopLevelConstants(SENode * node)106 std::vector<SEConstantNode*> GetAllTopLevelConstants(SENode* node) {
107   auto nodes = std::vector<SEConstantNode*>{};
108   if (auto recurrent_node = node->AsSEConstantNode()) {
109     nodes.push_back(recurrent_node);
110   }
111 
112   if (auto add_node = node->AsSEAddNode()) {
113     for (auto child : add_node->GetChildren()) {
114       auto child_nodes = GetAllTopLevelConstants(child);
115       nodes.insert(nodes.end(), child_nodes.begin(), child_nodes.end());
116     }
117   }
118 
119   return nodes;
120 }
121 
AreOffsetsAndCoefficientsConstant(const std::vector<SERecurrentNode * > & nodes)122 bool AreOffsetsAndCoefficientsConstant(
123     const std::vector<SERecurrentNode*>& nodes) {
124   for (auto node : nodes) {
125     if (!node->GetOffset()->AsSEConstantNode() ||
126         !node->GetOffset()->AsSEConstantNode()) {
127       return false;
128     }
129   }
130   return true;
131 }
132 
133 // Fold all SEConstantNode that appear in |recurrences| and |constants| into a
134 // single integer value.
CalculateConstantTerm(const std::vector<SERecurrentNode * > & recurrences,const std::vector<SEConstantNode * > & constants)135 int64_t CalculateConstantTerm(const std::vector<SERecurrentNode*>& recurrences,
136                               const std::vector<SEConstantNode*>& constants) {
137   int64_t constant_term = 0;
138   for (auto recurrence : recurrences) {
139     constant_term +=
140         recurrence->GetOffset()->AsSEConstantNode()->FoldToSingleValue();
141   }
142 
143   for (auto constant : constants) {
144     constant_term += constant->FoldToSingleValue();
145   }
146 
147   return constant_term;
148 }
149 
CalculateGCDFromCoefficients(const std::vector<SERecurrentNode * > & recurrences,int64_t running_gcd)150 int64_t CalculateGCDFromCoefficients(
151     const std::vector<SERecurrentNode*>& recurrences, int64_t running_gcd) {
152   for (SERecurrentNode* recurrence : recurrences) {
153     auto coefficient = recurrence->GetCoefficient()->AsSEConstantNode();
154 
155     running_gcd = GreatestCommonDivisor(
156         running_gcd, std::abs(coefficient->FoldToSingleValue()));
157   }
158 
159   return running_gcd;
160 }
161 
162 // Compare 2 fractions while first normalizing them, e.g. 2/4 and 4/8 will both
163 // be simplified to 1/2 and then determined to be equal.
NormalizeAndCompareFractions(int64_t numerator_0,int64_t denominator_0,int64_t numerator_1,int64_t denominator_1)164 bool NormalizeAndCompareFractions(int64_t numerator_0, int64_t denominator_0,
165                                   int64_t numerator_1, int64_t denominator_1) {
166   auto gcd_0 =
167       GreatestCommonDivisor(std::abs(numerator_0), std::abs(denominator_0));
168   auto gcd_1 =
169       GreatestCommonDivisor(std::abs(numerator_1), std::abs(denominator_1));
170 
171   auto normalized_numerator_0 = numerator_0 / gcd_0;
172   auto normalized_denominator_0 = denominator_0 / gcd_0;
173   auto normalized_numerator_1 = numerator_1 / gcd_1;
174   auto normalized_denominator_1 = denominator_1 / gcd_1;
175 
176   return normalized_numerator_0 == normalized_numerator_1 &&
177          normalized_denominator_0 == normalized_denominator_1;
178 }
179 
180 }  // namespace
181 
GetDependence(const Instruction * source,const Instruction * destination,DistanceVector * distance_vector)182 bool LoopDependenceAnalysis::GetDependence(const Instruction* source,
183                                            const Instruction* destination,
184                                            DistanceVector* distance_vector) {
185   // Start off by finding and marking all the loops in |loops_| that are
186   // irrelevant to the dependence analysis.
187   MarkUnsusedDistanceEntriesAsIrrelevant(source, destination, distance_vector);
188 
189   Instruction* source_access_chain = GetOperandDefinition(source, 0);
190   Instruction* destination_access_chain = GetOperandDefinition(destination, 0);
191 
192   auto num_access_chains =
193       (source_access_chain->opcode() == spv::Op::OpAccessChain) +
194       (destination_access_chain->opcode() == spv::Op::OpAccessChain);
195 
196   // If neither is an access chain, then they are load/store to a variable.
197   if (num_access_chains == 0) {
198     if (source_access_chain != destination_access_chain) {
199       // Not the same location, report independence
200       return true;
201     } else {
202       // Accessing the same variable
203       for (auto& entry : distance_vector->GetEntries()) {
204         entry = DistanceEntry();
205       }
206       return false;
207     }
208   }
209 
210   // If only one is an access chain, it could be accessing a part of a struct
211   if (num_access_chains == 1) {
212     auto source_is_chain =
213         source_access_chain->opcode() == spv::Op::OpAccessChain;
214     auto access_chain =
215         source_is_chain ? source_access_chain : destination_access_chain;
216     auto variable =
217         source_is_chain ? destination_access_chain : source_access_chain;
218 
219     auto location_in_chain = GetOperandDefinition(access_chain, 0);
220 
221     if (variable != location_in_chain) {
222       // Not the same location, report independence
223       return true;
224     } else {
225       // Accessing the same variable
226       for (auto& entry : distance_vector->GetEntries()) {
227         entry = DistanceEntry();
228       }
229       return false;
230     }
231   }
232 
233   // If the access chains aren't collecting from the same structure there is no
234   // dependence.
235   Instruction* source_array = GetOperandDefinition(source_access_chain, 0);
236   Instruction* destination_array =
237       GetOperandDefinition(destination_access_chain, 0);
238 
239   // Nested access chains are not supported yet, bail out.
240   if (source_array->opcode() == spv::Op::OpAccessChain ||
241       destination_array->opcode() == spv::Op::OpAccessChain) {
242     for (auto& entry : distance_vector->GetEntries()) {
243       entry = DistanceEntry();
244     }
245     return false;
246   }
247 
248   if (source_array != destination_array) {
249     PrintDebug("Proved independence through different arrays.");
250     return true;
251   }
252 
253   // To handle multiple subscripts we must get every operand in the access
254   // chains past the first.
255   std::vector<Instruction*> source_subscripts = GetSubscripts(source);
256   std::vector<Instruction*> destination_subscripts = GetSubscripts(destination);
257 
258   auto sets_of_subscripts =
259       PartitionSubscripts(source_subscripts, destination_subscripts);
260 
261   auto first_coupled = std::partition(
262       std::begin(sets_of_subscripts), std::end(sets_of_subscripts),
263       [](const std::set<std::pair<Instruction*, Instruction*>>& set) {
264         return set.size() == 1;
265       });
266 
267   // Go through each subscript testing for independence.
268   // If any subscript results in independence, we prove independence between the
269   // load and store.
270   // If we can't prove independence we store what information we can gather in
271   // a DistanceVector.
272   for (auto it = std::begin(sets_of_subscripts); it < first_coupled; ++it) {
273     auto source_subscript = std::get<0>(*(*it).begin());
274     auto destination_subscript = std::get<1>(*(*it).begin());
275 
276     SENode* source_node = scalar_evolution_.SimplifyExpression(
277         scalar_evolution_.AnalyzeInstruction(source_subscript));
278     SENode* destination_node = scalar_evolution_.SimplifyExpression(
279         scalar_evolution_.AnalyzeInstruction(destination_subscript));
280 
281     // Check the loops are in a form we support.
282     auto subscript_pair = std::make_pair(source_node, destination_node);
283 
284     const Loop* loop = GetLoopForSubscriptPair(subscript_pair);
285     if (loop) {
286       if (!IsSupportedLoop(loop)) {
287         PrintDebug(
288             "GetDependence found an unsupported loop form. Assuming <=> for "
289             "loop.");
290         DistanceEntry* distance_entry =
291             GetDistanceEntryForSubscriptPair(subscript_pair, distance_vector);
292         if (distance_entry) {
293           distance_entry->direction = DistanceEntry::Directions::ALL;
294         }
295         continue;
296       }
297     }
298 
299     // If either node is simplified to a CanNotCompute we can't perform any
300     // analysis so must assume <=> dependence and return.
301     if (source_node->GetType() == SENode::CanNotCompute ||
302         destination_node->GetType() == SENode::CanNotCompute) {
303       // Record the <=> dependence if we can get a DistanceEntry
304       PrintDebug(
305           "GetDependence found source_node || destination_node as "
306           "CanNotCompute. Abandoning evaluation for this subscript.");
307       DistanceEntry* distance_entry =
308           GetDistanceEntryForSubscriptPair(subscript_pair, distance_vector);
309       if (distance_entry) {
310         distance_entry->direction = DistanceEntry::Directions::ALL;
311       }
312       continue;
313     }
314 
315     // We have no induction variables so can apply a ZIV test.
316     if (IsZIV(subscript_pair)) {
317       PrintDebug("Found a ZIV subscript pair");
318       if (ZIVTest(subscript_pair)) {
319         PrintDebug("Proved independence with ZIVTest.");
320         return true;
321       }
322     }
323 
324     // We have only one induction variable so should attempt an SIV test.
325     if (IsSIV(subscript_pair)) {
326       PrintDebug("Found a SIV subscript pair.");
327       if (SIVTest(subscript_pair, distance_vector)) {
328         PrintDebug("Proved independence with SIVTest.");
329         return true;
330       }
331     }
332 
333     // We have multiple induction variables so should attempt an MIV test.
334     if (IsMIV(subscript_pair)) {
335       PrintDebug("Found a MIV subscript pair.");
336       if (GCDMIVTest(subscript_pair)) {
337         PrintDebug("Proved independence with the GCD test.");
338         auto current_loops = CollectLoops(source_node, destination_node);
339 
340         for (auto current_loop : current_loops) {
341           auto distance_entry =
342               GetDistanceEntryForLoop(current_loop, distance_vector);
343           distance_entry->direction = DistanceEntry::Directions::NONE;
344         }
345         return true;
346       }
347     }
348   }
349 
350   for (auto it = first_coupled; it < std::end(sets_of_subscripts); ++it) {
351     auto coupled_instructions = *it;
352     std::vector<SubscriptPair> coupled_subscripts{};
353 
354     for (const auto& elem : coupled_instructions) {
355       auto source_subscript = std::get<0>(elem);
356       auto destination_subscript = std::get<1>(elem);
357 
358       SENode* source_node = scalar_evolution_.SimplifyExpression(
359           scalar_evolution_.AnalyzeInstruction(source_subscript));
360       SENode* destination_node = scalar_evolution_.SimplifyExpression(
361           scalar_evolution_.AnalyzeInstruction(destination_subscript));
362 
363       coupled_subscripts.push_back({source_node, destination_node});
364     }
365 
366     auto supported = true;
367 
368     for (const auto& subscript : coupled_subscripts) {
369       auto loops = CollectLoops(std::get<0>(subscript), std::get<1>(subscript));
370 
371       auto is_subscript_supported =
372           std::all_of(std::begin(loops), std::end(loops),
373                       [this](const Loop* l) { return IsSupportedLoop(l); });
374 
375       supported = supported && is_subscript_supported;
376     }
377 
378     if (DeltaTest(coupled_subscripts, distance_vector)) {
379       return true;
380     }
381   }
382 
383   // We were unable to prove independence so must gather all of the direction
384   // information we found.
385   PrintDebug(
386       "Couldn't prove independence.\n"
387       "All possible direction information has been collected in the input "
388       "DistanceVector.");
389 
390   return false;
391 }
392 
ZIVTest(const std::pair<SENode *,SENode * > & subscript_pair)393 bool LoopDependenceAnalysis::ZIVTest(
394     const std::pair<SENode*, SENode*>& subscript_pair) {
395   auto source = std::get<0>(subscript_pair);
396   auto destination = std::get<1>(subscript_pair);
397 
398   PrintDebug("Performing ZIVTest");
399   // If source == destination, dependence with direction = and distance 0.
400   if (source == destination) {
401     PrintDebug("ZIVTest found EQ dependence.");
402     return false;
403   } else {
404     PrintDebug("ZIVTest found independence.");
405     // Otherwise we prove independence.
406     return true;
407   }
408 }
409 
SIVTest(const std::pair<SENode *,SENode * > & subscript_pair,DistanceVector * distance_vector)410 bool LoopDependenceAnalysis::SIVTest(
411     const std::pair<SENode*, SENode*>& subscript_pair,
412     DistanceVector* distance_vector) {
413   DistanceEntry* distance_entry =
414       GetDistanceEntryForSubscriptPair(subscript_pair, distance_vector);
415   if (!distance_entry) {
416     PrintDebug(
417         "SIVTest could not find a DistanceEntry for subscript_pair. Exiting");
418   }
419 
420   SENode* source_node = std::get<0>(subscript_pair);
421   SENode* destination_node = std::get<1>(subscript_pair);
422 
423   int64_t source_induction_count = CountInductionVariables(source_node);
424   int64_t destination_induction_count =
425       CountInductionVariables(destination_node);
426 
427   // If the source node has no induction variables we can apply a
428   // WeakZeroSrcTest.
429   if (source_induction_count == 0) {
430     PrintDebug("Found source has no induction variable.");
431     if (WeakZeroSourceSIVTest(
432             source_node, destination_node->AsSERecurrentNode(),
433             destination_node->AsSERecurrentNode()->GetCoefficient(),
434             distance_entry)) {
435       PrintDebug("Proved independence with WeakZeroSourceSIVTest.");
436       distance_entry->dependence_information =
437           DistanceEntry::DependenceInformation::DIRECTION;
438       distance_entry->direction = DistanceEntry::Directions::NONE;
439       return true;
440     }
441   }
442 
443   // If the destination has no induction variables we can apply a
444   // WeakZeroDestTest.
445   if (destination_induction_count == 0) {
446     PrintDebug("Found destination has no induction variable.");
447     if (WeakZeroDestinationSIVTest(
448             source_node->AsSERecurrentNode(), destination_node,
449             source_node->AsSERecurrentNode()->GetCoefficient(),
450             distance_entry)) {
451       PrintDebug("Proved independence with WeakZeroDestinationSIVTest.");
452       distance_entry->dependence_information =
453           DistanceEntry::DependenceInformation::DIRECTION;
454       distance_entry->direction = DistanceEntry::Directions::NONE;
455       return true;
456     }
457   }
458 
459   // We now need to collect the SERecurrentExpr nodes from source and
460   // destination. We do not handle cases where source or destination have
461   // multiple SERecurrentExpr nodes.
462   std::vector<SERecurrentNode*> source_recurrent_nodes =
463       source_node->CollectRecurrentNodes();
464   std::vector<SERecurrentNode*> destination_recurrent_nodes =
465       destination_node->CollectRecurrentNodes();
466 
467   if (source_recurrent_nodes.size() == 1 &&
468       destination_recurrent_nodes.size() == 1) {
469     PrintDebug("Found source and destination have 1 induction variable.");
470     SERecurrentNode* source_recurrent_expr = *source_recurrent_nodes.begin();
471     SERecurrentNode* destination_recurrent_expr =
472         *destination_recurrent_nodes.begin();
473 
474     // If the coefficients are identical we can apply a StrongSIVTest.
475     if (source_recurrent_expr->GetCoefficient() ==
476         destination_recurrent_expr->GetCoefficient()) {
477       PrintDebug("Found source and destination share coefficient.");
478       if (StrongSIVTest(source_node, destination_node,
479                         source_recurrent_expr->GetCoefficient(),
480                         distance_entry)) {
481         PrintDebug("Proved independence with StrongSIVTest");
482         distance_entry->dependence_information =
483             DistanceEntry::DependenceInformation::DIRECTION;
484         distance_entry->direction = DistanceEntry::Directions::NONE;
485         return true;
486       }
487     }
488 
489     // If the coefficients are of equal magnitude and opposite sign we can
490     // apply a WeakCrossingSIVTest.
491     if (source_recurrent_expr->GetCoefficient() ==
492         scalar_evolution_.CreateNegation(
493             destination_recurrent_expr->GetCoefficient())) {
494       PrintDebug("Found source coefficient = -destination coefficient.");
495       if (WeakCrossingSIVTest(source_node, destination_node,
496                               source_recurrent_expr->GetCoefficient(),
497                               distance_entry)) {
498         PrintDebug("Proved independence with WeakCrossingSIVTest");
499         distance_entry->dependence_information =
500             DistanceEntry::DependenceInformation::DIRECTION;
501         distance_entry->direction = DistanceEntry::Directions::NONE;
502         return true;
503       }
504     }
505   }
506 
507   return false;
508 }
509 
StrongSIVTest(SENode * source,SENode * destination,SENode * coefficient,DistanceEntry * distance_entry)510 bool LoopDependenceAnalysis::StrongSIVTest(SENode* source, SENode* destination,
511                                            SENode* coefficient,
512                                            DistanceEntry* distance_entry) {
513   PrintDebug("Performing StrongSIVTest.");
514   // If both source and destination are SERecurrentNodes we can perform tests
515   // based on distance.
516   // If either source or destination contain value unknown nodes or if one or
517   // both are not SERecurrentNodes we must attempt a symbolic test.
518   std::vector<SEValueUnknown*> source_value_unknown_nodes =
519       source->CollectValueUnknownNodes();
520   std::vector<SEValueUnknown*> destination_value_unknown_nodes =
521       destination->CollectValueUnknownNodes();
522   if (source_value_unknown_nodes.size() > 0 ||
523       destination_value_unknown_nodes.size() > 0) {
524     PrintDebug(
525         "StrongSIVTest found symbolics. Will attempt SymbolicStrongSIVTest.");
526     return SymbolicStrongSIVTest(source, destination, coefficient,
527                                  distance_entry);
528   }
529 
530   if (!source->AsSERecurrentNode() || !destination->AsSERecurrentNode()) {
531     PrintDebug(
532         "StrongSIVTest could not simplify source and destination to "
533         "SERecurrentNodes so will exit.");
534     distance_entry->direction = DistanceEntry::Directions::ALL;
535     return false;
536   }
537 
538   // Build an SENode for distance.
539   std::pair<SENode*, SENode*> subscript_pair =
540       std::make_pair(source, destination);
541   const Loop* subscript_loop = GetLoopForSubscriptPair(subscript_pair);
542   SENode* source_constant_term =
543       GetConstantTerm(subscript_loop, source->AsSERecurrentNode());
544   SENode* destination_constant_term =
545       GetConstantTerm(subscript_loop, destination->AsSERecurrentNode());
546   if (!source_constant_term || !destination_constant_term) {
547     PrintDebug(
548         "StrongSIVTest could not collect the constant terms of either source "
549         "or destination so will exit.");
550     return false;
551   }
552   SENode* constant_term_delta =
553       scalar_evolution_.SimplifyExpression(scalar_evolution_.CreateSubtraction(
554           destination_constant_term, source_constant_term));
555 
556   // Scalar evolution doesn't perform division, so we must fold to constants and
557   // do it manually.
558   // We must check the offset delta and coefficient are constants.
559   int64_t distance = 0;
560   SEConstantNode* delta_constant = constant_term_delta->AsSEConstantNode();
561   SEConstantNode* coefficient_constant = coefficient->AsSEConstantNode();
562   if (delta_constant && coefficient_constant) {
563     int64_t delta_value = delta_constant->FoldToSingleValue();
564     int64_t coefficient_value = coefficient_constant->FoldToSingleValue();
565     PrintDebug(
566         "StrongSIVTest found delta value and coefficient value as constants "
567         "with values:\n"
568         "\tdelta value: " +
569         ToString(delta_value) +
570         "\n\tcoefficient value: " + ToString(coefficient_value) + "\n");
571     // Check if the distance is not integral to try to prove independence.
572     if (delta_value % coefficient_value != 0) {
573       PrintDebug(
574           "StrongSIVTest proved independence through distance not being an "
575           "integer.");
576       distance_entry->dependence_information =
577           DistanceEntry::DependenceInformation::DIRECTION;
578       distance_entry->direction = DistanceEntry::Directions::NONE;
579       return true;
580     } else {
581       distance = delta_value / coefficient_value;
582       PrintDebug("StrongSIV test found distance as " + ToString(distance));
583     }
584   } else {
585     // If we can't fold delta and coefficient to single values we can't produce
586     // distance.
587     // As a result we can't perform the rest of the pass and must assume
588     // dependence in all directions.
589     PrintDebug("StrongSIVTest could not produce a distance. Must exit.");
590     distance_entry->distance = DistanceEntry::Directions::ALL;
591     return false;
592   }
593 
594   // Next we gather the upper and lower bounds as constants if possible. If
595   // distance > upper_bound - lower_bound we prove independence.
596   SENode* lower_bound = GetLowerBound(subscript_loop);
597   SENode* upper_bound = GetUpperBound(subscript_loop);
598   if (lower_bound && upper_bound) {
599     PrintDebug("StrongSIVTest found bounds.");
600     SENode* bounds = scalar_evolution_.SimplifyExpression(
601         scalar_evolution_.CreateSubtraction(upper_bound, lower_bound));
602 
603     if (bounds->GetType() == SENode::SENodeType::Constant) {
604       int64_t bounds_value = bounds->AsSEConstantNode()->FoldToSingleValue();
605       PrintDebug(
606           "StrongSIVTest found upper_bound - lower_bound as a constant with "
607           "value " +
608           ToString(bounds_value));
609 
610       // If the absolute value of the distance is > upper bound - lower bound
611       // then we prove independence.
612       if (llabs(distance) > llabs(bounds_value)) {
613         PrintDebug(
614             "StrongSIVTest proved independence through distance escaping the "
615             "loop bounds.");
616         distance_entry->dependence_information =
617             DistanceEntry::DependenceInformation::DISTANCE;
618         distance_entry->direction = DistanceEntry::Directions::NONE;
619         distance_entry->distance = distance;
620         return true;
621       }
622     }
623   } else {
624     PrintDebug("StrongSIVTest was unable to gather lower and upper bounds.");
625   }
626 
627   // Otherwise we can get a direction as follows
628   //             { < if distance > 0
629   // direction = { = if distance == 0
630   //             { > if distance < 0
631   PrintDebug(
632       "StrongSIVTest could not prove independence. Gathering direction "
633       "information.");
634   if (distance > 0) {
635     distance_entry->dependence_information =
636         DistanceEntry::DependenceInformation::DISTANCE;
637     distance_entry->direction = DistanceEntry::Directions::LT;
638     distance_entry->distance = distance;
639     return false;
640   }
641   if (distance == 0) {
642     distance_entry->dependence_information =
643         DistanceEntry::DependenceInformation::DISTANCE;
644     distance_entry->direction = DistanceEntry::Directions::EQ;
645     distance_entry->distance = 0;
646     return false;
647   }
648   if (distance < 0) {
649     distance_entry->dependence_information =
650         DistanceEntry::DependenceInformation::DISTANCE;
651     distance_entry->direction = DistanceEntry::Directions::GT;
652     distance_entry->distance = distance;
653     return false;
654   }
655 
656   // We were unable to prove independence or discern any additional information
657   // Must assume <=> direction.
658   PrintDebug(
659       "StrongSIVTest was unable to determine any dependence information.");
660   distance_entry->direction = DistanceEntry::Directions::ALL;
661   return false;
662 }
663 
SymbolicStrongSIVTest(SENode * source,SENode * destination,SENode * coefficient,DistanceEntry * distance_entry)664 bool LoopDependenceAnalysis::SymbolicStrongSIVTest(
665     SENode* source, SENode* destination, SENode* coefficient,
666     DistanceEntry* distance_entry) {
667   PrintDebug("Performing SymbolicStrongSIVTest.");
668   SENode* source_destination_delta = scalar_evolution_.SimplifyExpression(
669       scalar_evolution_.CreateSubtraction(source, destination));
670   // By cancelling out the induction variables by subtracting the source and
671   // destination we can produce an expression of symbolics and constants. This
672   // expression can be compared to the loop bounds to find if the offset is
673   // outwith the bounds.
674   std::pair<SENode*, SENode*> subscript_pair =
675       std::make_pair(source, destination);
676   const Loop* subscript_loop = GetLoopForSubscriptPair(subscript_pair);
677   if (IsProvablyOutsideOfLoopBounds(subscript_loop, source_destination_delta,
678                                     coefficient)) {
679     PrintDebug(
680         "SymbolicStrongSIVTest proved independence through loop bounds.");
681     distance_entry->dependence_information =
682         DistanceEntry::DependenceInformation::DIRECTION;
683     distance_entry->direction = DistanceEntry::Directions::NONE;
684     return true;
685   }
686   // We were unable to prove independence or discern any additional information.
687   // Must assume <=> direction.
688   PrintDebug(
689       "SymbolicStrongSIVTest was unable to determine any dependence "
690       "information.");
691   distance_entry->direction = DistanceEntry::Directions::ALL;
692   return false;
693 }
694 
WeakZeroSourceSIVTest(SENode * source,SERecurrentNode * destination,SENode * coefficient,DistanceEntry * distance_entry)695 bool LoopDependenceAnalysis::WeakZeroSourceSIVTest(
696     SENode* source, SERecurrentNode* destination, SENode* coefficient,
697     DistanceEntry* distance_entry) {
698   PrintDebug("Performing WeakZeroSourceSIVTest.");
699   std::pair<SENode*, SENode*> subscript_pair =
700       std::make_pair(source, destination);
701   const Loop* subscript_loop = GetLoopForSubscriptPair(subscript_pair);
702   // Build an SENode for distance.
703   SENode* destination_constant_term =
704       GetConstantTerm(subscript_loop, destination);
705   SENode* delta = scalar_evolution_.SimplifyExpression(
706       scalar_evolution_.CreateSubtraction(source, destination_constant_term));
707 
708   // Scalar evolution doesn't perform division, so we must fold to constants and
709   // do it manually.
710   int64_t distance = 0;
711   SEConstantNode* delta_constant = delta->AsSEConstantNode();
712   SEConstantNode* coefficient_constant = coefficient->AsSEConstantNode();
713   if (delta_constant && coefficient_constant) {
714     PrintDebug(
715         "WeakZeroSourceSIVTest folding delta and coefficient to constants.");
716     int64_t delta_value = delta_constant->FoldToSingleValue();
717     int64_t coefficient_value = coefficient_constant->FoldToSingleValue();
718     // Check if the distance is not integral.
719     if (delta_value % coefficient_value != 0) {
720       PrintDebug(
721           "WeakZeroSourceSIVTest proved independence through distance not "
722           "being an integer.");
723       distance_entry->dependence_information =
724           DistanceEntry::DependenceInformation::DIRECTION;
725       distance_entry->direction = DistanceEntry::Directions::NONE;
726       return true;
727     } else {
728       distance = delta_value / coefficient_value;
729       PrintDebug(
730           "WeakZeroSourceSIVTest calculated distance with the following "
731           "values\n"
732           "\tdelta value: " +
733           ToString(delta_value) +
734           "\n\tcoefficient value: " + ToString(coefficient_value) +
735           "\n\tdistance: " + ToString(distance) + "\n");
736     }
737   } else {
738     PrintDebug(
739         "WeakZeroSourceSIVTest was unable to fold delta and coefficient to "
740         "constants.");
741   }
742 
743   // If we can prove the distance is outside the bounds we prove independence.
744   SEConstantNode* lower_bound =
745       GetLowerBound(subscript_loop)->AsSEConstantNode();
746   SEConstantNode* upper_bound =
747       GetUpperBound(subscript_loop)->AsSEConstantNode();
748   if (lower_bound && upper_bound) {
749     PrintDebug("WeakZeroSourceSIVTest found bounds as SEConstantNodes.");
750     int64_t lower_bound_value = lower_bound->FoldToSingleValue();
751     int64_t upper_bound_value = upper_bound->FoldToSingleValue();
752     if (!IsWithinBounds(llabs(distance), lower_bound_value,
753                         upper_bound_value)) {
754       PrintDebug(
755           "WeakZeroSourceSIVTest proved independence through distance escaping "
756           "the loop bounds.");
757       PrintDebug(
758           "Bound values were as follow\n"
759           "\tlower bound value: " +
760           ToString(lower_bound_value) +
761           "\n\tupper bound value: " + ToString(upper_bound_value) +
762           "\n\tdistance value: " + ToString(distance) + "\n");
763       distance_entry->dependence_information =
764           DistanceEntry::DependenceInformation::DISTANCE;
765       distance_entry->direction = DistanceEntry::Directions::NONE;
766       distance_entry->distance = distance;
767       return true;
768     }
769   } else {
770     PrintDebug(
771         "WeakZeroSourceSIVTest was unable to find lower and upper bound as "
772         "SEConstantNodes.");
773   }
774 
775   // Now we want to see if we can detect to peel the first or last iterations.
776 
777   // We get the FirstTripValue as GetFirstTripInductionNode() +
778   // GetConstantTerm(destination)
779   SENode* first_trip_SENode =
780       scalar_evolution_.SimplifyExpression(scalar_evolution_.CreateAddNode(
781           GetFirstTripInductionNode(subscript_loop),
782           GetConstantTerm(subscript_loop, destination)));
783 
784   // If source == FirstTripValue, peel_first.
785   if (first_trip_SENode) {
786     PrintDebug("WeakZeroSourceSIVTest built first_trip_SENode.");
787     if (first_trip_SENode->AsSEConstantNode()) {
788       PrintDebug(
789           "WeakZeroSourceSIVTest has found first_trip_SENode as an "
790           "SEConstantNode with value: " +
791           ToString(first_trip_SENode->AsSEConstantNode()->FoldToSingleValue()) +
792           "\n");
793     }
794     if (source == first_trip_SENode) {
795       // We have found that peeling the first iteration will break dependency.
796       PrintDebug(
797           "WeakZeroSourceSIVTest has found peeling first iteration will break "
798           "dependency");
799       distance_entry->dependence_information =
800           DistanceEntry::DependenceInformation::PEEL;
801       distance_entry->peel_first = true;
802       return false;
803     }
804   } else {
805     PrintDebug("WeakZeroSourceSIVTest was unable to build first_trip_SENode");
806   }
807 
808   // We get the LastTripValue as GetFinalTripInductionNode(coefficient) +
809   // GetConstantTerm(destination)
810   SENode* final_trip_SENode =
811       scalar_evolution_.SimplifyExpression(scalar_evolution_.CreateAddNode(
812           GetFinalTripInductionNode(subscript_loop, coefficient),
813           GetConstantTerm(subscript_loop, destination)));
814 
815   // If source == LastTripValue, peel_last.
816   if (final_trip_SENode) {
817     PrintDebug("WeakZeroSourceSIVTest built final_trip_SENode.");
818     if (first_trip_SENode->AsSEConstantNode()) {
819       PrintDebug(
820           "WeakZeroSourceSIVTest has found final_trip_SENode as an "
821           "SEConstantNode with value: " +
822           ToString(final_trip_SENode->AsSEConstantNode()->FoldToSingleValue()) +
823           "\n");
824     }
825     if (source == final_trip_SENode) {
826       // We have found that peeling the last iteration will break dependency.
827       PrintDebug(
828           "WeakZeroSourceSIVTest has found peeling final iteration will break "
829           "dependency");
830       distance_entry->dependence_information =
831           DistanceEntry::DependenceInformation::PEEL;
832       distance_entry->peel_last = true;
833       return false;
834     }
835   } else {
836     PrintDebug("WeakZeroSourceSIVTest was unable to build final_trip_SENode");
837   }
838 
839   // We were unable to prove independence or discern any additional information.
840   // Must assume <=> direction.
841   PrintDebug(
842       "WeakZeroSourceSIVTest was unable to determine any dependence "
843       "information.");
844   distance_entry->direction = DistanceEntry::Directions::ALL;
845   return false;
846 }
847 
WeakZeroDestinationSIVTest(SERecurrentNode * source,SENode * destination,SENode * coefficient,DistanceEntry * distance_entry)848 bool LoopDependenceAnalysis::WeakZeroDestinationSIVTest(
849     SERecurrentNode* source, SENode* destination, SENode* coefficient,
850     DistanceEntry* distance_entry) {
851   PrintDebug("Performing WeakZeroDestinationSIVTest.");
852   // Build an SENode for distance.
853   std::pair<SENode*, SENode*> subscript_pair =
854       std::make_pair(source, destination);
855   const Loop* subscript_loop = GetLoopForSubscriptPair(subscript_pair);
856   SENode* source_constant_term = GetConstantTerm(subscript_loop, source);
857   SENode* delta = scalar_evolution_.SimplifyExpression(
858       scalar_evolution_.CreateSubtraction(destination, source_constant_term));
859 
860   // Scalar evolution doesn't perform division, so we must fold to constants and
861   // do it manually.
862   int64_t distance = 0;
863   SEConstantNode* delta_constant = delta->AsSEConstantNode();
864   SEConstantNode* coefficient_constant = coefficient->AsSEConstantNode();
865   if (delta_constant && coefficient_constant) {
866     PrintDebug(
867         "WeakZeroDestinationSIVTest folding delta and coefficient to "
868         "constants.");
869     int64_t delta_value = delta_constant->FoldToSingleValue();
870     int64_t coefficient_value = coefficient_constant->FoldToSingleValue();
871     // Check if the distance is not integral.
872     if (delta_value % coefficient_value != 0) {
873       PrintDebug(
874           "WeakZeroDestinationSIVTest proved independence through distance not "
875           "being an integer.");
876       distance_entry->dependence_information =
877           DistanceEntry::DependenceInformation::DIRECTION;
878       distance_entry->direction = DistanceEntry::Directions::NONE;
879       return true;
880     } else {
881       distance = delta_value / coefficient_value;
882       PrintDebug(
883           "WeakZeroDestinationSIVTest calculated distance with the following "
884           "values\n"
885           "\tdelta value: " +
886           ToString(delta_value) +
887           "\n\tcoefficient value: " + ToString(coefficient_value) +
888           "\n\tdistance: " + ToString(distance) + "\n");
889     }
890   } else {
891     PrintDebug(
892         "WeakZeroDestinationSIVTest was unable to fold delta and coefficient "
893         "to constants.");
894   }
895 
896   // If we can prove the distance is outside the bounds we prove independence.
897   SEConstantNode* lower_bound =
898       GetLowerBound(subscript_loop)->AsSEConstantNode();
899   SEConstantNode* upper_bound =
900       GetUpperBound(subscript_loop)->AsSEConstantNode();
901   if (lower_bound && upper_bound) {
902     PrintDebug("WeakZeroDestinationSIVTest found bounds as SEConstantNodes.");
903     int64_t lower_bound_value = lower_bound->FoldToSingleValue();
904     int64_t upper_bound_value = upper_bound->FoldToSingleValue();
905     if (!IsWithinBounds(llabs(distance), lower_bound_value,
906                         upper_bound_value)) {
907       PrintDebug(
908           "WeakZeroDestinationSIVTest proved independence through distance "
909           "escaping the loop bounds.");
910       PrintDebug(
911           "Bound values were as follows\n"
912           "\tlower bound value: " +
913           ToString(lower_bound_value) +
914           "\n\tupper bound value: " + ToString(upper_bound_value) +
915           "\n\tdistance value: " + ToString(distance));
916       distance_entry->dependence_information =
917           DistanceEntry::DependenceInformation::DISTANCE;
918       distance_entry->direction = DistanceEntry::Directions::NONE;
919       distance_entry->distance = distance;
920       return true;
921     }
922   } else {
923     PrintDebug(
924         "WeakZeroDestinationSIVTest was unable to find lower and upper bound "
925         "as SEConstantNodes.");
926   }
927 
928   // Now we want to see if we can detect to peel the first or last iterations.
929 
930   // We get the FirstTripValue as GetFirstTripInductionNode() +
931   // GetConstantTerm(source)
932   SENode* first_trip_SENode = scalar_evolution_.SimplifyExpression(
933       scalar_evolution_.CreateAddNode(GetFirstTripInductionNode(subscript_loop),
934                                       GetConstantTerm(subscript_loop, source)));
935 
936   // If destination == FirstTripValue, peel_first.
937   if (first_trip_SENode) {
938     PrintDebug("WeakZeroDestinationSIVTest built first_trip_SENode.");
939     if (first_trip_SENode->AsSEConstantNode()) {
940       PrintDebug(
941           "WeakZeroDestinationSIVTest has found first_trip_SENode as an "
942           "SEConstantNode with value: " +
943           ToString(first_trip_SENode->AsSEConstantNode()->FoldToSingleValue()) +
944           "\n");
945     }
946     if (destination == first_trip_SENode) {
947       // We have found that peeling the first iteration will break dependency.
948       PrintDebug(
949           "WeakZeroDestinationSIVTest has found peeling first iteration will "
950           "break dependency");
951       distance_entry->dependence_information =
952           DistanceEntry::DependenceInformation::PEEL;
953       distance_entry->peel_first = true;
954       return false;
955     }
956   } else {
957     PrintDebug(
958         "WeakZeroDestinationSIVTest was unable to build first_trip_SENode");
959   }
960 
961   // We get the LastTripValue as GetFinalTripInductionNode(coefficient) +
962   // GetConstantTerm(source)
963   SENode* final_trip_SENode =
964       scalar_evolution_.SimplifyExpression(scalar_evolution_.CreateAddNode(
965           GetFinalTripInductionNode(subscript_loop, coefficient),
966           GetConstantTerm(subscript_loop, source)));
967 
968   // If destination == LastTripValue, peel_last.
969   if (final_trip_SENode) {
970     PrintDebug("WeakZeroDestinationSIVTest built final_trip_SENode.");
971     if (final_trip_SENode->AsSEConstantNode()) {
972       PrintDebug(
973           "WeakZeroDestinationSIVTest has found final_trip_SENode as an "
974           "SEConstantNode with value: " +
975           ToString(final_trip_SENode->AsSEConstantNode()->FoldToSingleValue()) +
976           "\n");
977     }
978     if (destination == final_trip_SENode) {
979       // We have found that peeling the last iteration will break dependency.
980       PrintDebug(
981           "WeakZeroDestinationSIVTest has found peeling final iteration will "
982           "break dependency");
983       distance_entry->dependence_information =
984           DistanceEntry::DependenceInformation::PEEL;
985       distance_entry->peel_last = true;
986       return false;
987     }
988   } else {
989     PrintDebug(
990         "WeakZeroDestinationSIVTest was unable to build final_trip_SENode");
991   }
992 
993   // We were unable to prove independence or discern any additional information.
994   // Must assume <=> direction.
995   PrintDebug(
996       "WeakZeroDestinationSIVTest was unable to determine any dependence "
997       "information.");
998   distance_entry->direction = DistanceEntry::Directions::ALL;
999   return false;
1000 }
1001 
WeakCrossingSIVTest(SENode * source,SENode * destination,SENode * coefficient,DistanceEntry * distance_entry)1002 bool LoopDependenceAnalysis::WeakCrossingSIVTest(
1003     SENode* source, SENode* destination, SENode* coefficient,
1004     DistanceEntry* distance_entry) {
1005   PrintDebug("Performing WeakCrossingSIVTest.");
1006   // We currently can't handle symbolic WeakCrossingSIVTests. If either source
1007   // or destination are not SERecurrentNodes we must exit.
1008   if (!source->AsSERecurrentNode() || !destination->AsSERecurrentNode()) {
1009     PrintDebug(
1010         "WeakCrossingSIVTest found source or destination != SERecurrentNode. "
1011         "Exiting");
1012     distance_entry->direction = DistanceEntry::Directions::ALL;
1013     return false;
1014   }
1015 
1016   // Build an SENode for distance.
1017   SENode* offset_delta =
1018       scalar_evolution_.SimplifyExpression(scalar_evolution_.CreateSubtraction(
1019           destination->AsSERecurrentNode()->GetOffset(),
1020           source->AsSERecurrentNode()->GetOffset()));
1021 
1022   // Scalar evolution doesn't perform division, so we must fold to constants and
1023   // do it manually.
1024   int64_t distance = 0;
1025   SEConstantNode* delta_constant = offset_delta->AsSEConstantNode();
1026   SEConstantNode* coefficient_constant = coefficient->AsSEConstantNode();
1027   if (delta_constant && coefficient_constant) {
1028     PrintDebug(
1029         "WeakCrossingSIVTest folding offset_delta and coefficient to "
1030         "constants.");
1031     int64_t delta_value = delta_constant->FoldToSingleValue();
1032     int64_t coefficient_value = coefficient_constant->FoldToSingleValue();
1033     // Check if the distance is not integral or if it has a non-integral part
1034     // equal to 1/2.
1035     if (delta_value % (2 * coefficient_value) != 0 &&
1036         static_cast<float>(delta_value % (2 * coefficient_value)) /
1037                 static_cast<float>(2 * coefficient_value) !=
1038             0.5) {
1039       PrintDebug(
1040           "WeakCrossingSIVTest proved independence through distance escaping "
1041           "the loop bounds.");
1042       distance_entry->dependence_information =
1043           DistanceEntry::DependenceInformation::DIRECTION;
1044       distance_entry->direction = DistanceEntry::Directions::NONE;
1045       return true;
1046     } else {
1047       distance = delta_value / (2 * coefficient_value);
1048     }
1049 
1050     if (distance == 0) {
1051       PrintDebug("WeakCrossingSIVTest found EQ dependence.");
1052       distance_entry->dependence_information =
1053           DistanceEntry::DependenceInformation::DISTANCE;
1054       distance_entry->direction = DistanceEntry::Directions::EQ;
1055       distance_entry->distance = 0;
1056       return false;
1057     }
1058   } else {
1059     PrintDebug(
1060         "WeakCrossingSIVTest was unable to fold offset_delta and coefficient "
1061         "to constants.");
1062   }
1063 
1064   // We were unable to prove independence or discern any additional information.
1065   // Must assume <=> direction.
1066   PrintDebug(
1067       "WeakCrossingSIVTest was unable to determine any dependence "
1068       "information.");
1069   distance_entry->direction = DistanceEntry::Directions::ALL;
1070   return false;
1071 }
1072 
1073 // Perform the GCD test if both, the source and the destination nodes, are in
1074 // the form a0*i0 + a1*i1 + ... an*in + c.
GCDMIVTest(const std::pair<SENode *,SENode * > & subscript_pair)1075 bool LoopDependenceAnalysis::GCDMIVTest(
1076     const std::pair<SENode*, SENode*>& subscript_pair) {
1077   auto source = std::get<0>(subscript_pair);
1078   auto destination = std::get<1>(subscript_pair);
1079 
1080   // Bail out if source/destination is in an unexpected form.
1081   if (!IsInCorrectFormForGCDTest(source) ||
1082       !IsInCorrectFormForGCDTest(destination)) {
1083     return false;
1084   }
1085 
1086   auto source_recurrences = GetAllTopLevelRecurrences(source);
1087   auto dest_recurrences = GetAllTopLevelRecurrences(destination);
1088 
1089   // Bail out if all offsets and coefficients aren't constant.
1090   if (!AreOffsetsAndCoefficientsConstant(source_recurrences) ||
1091       !AreOffsetsAndCoefficientsConstant(dest_recurrences)) {
1092     return false;
1093   }
1094 
1095   // Calculate the GCD of all coefficients.
1096   auto source_constants = GetAllTopLevelConstants(source);
1097   int64_t source_constant =
1098       CalculateConstantTerm(source_recurrences, source_constants);
1099 
1100   auto dest_constants = GetAllTopLevelConstants(destination);
1101   int64_t destination_constant =
1102       CalculateConstantTerm(dest_recurrences, dest_constants);
1103 
1104   int64_t delta = std::abs(source_constant - destination_constant);
1105 
1106   int64_t running_gcd = 0;
1107 
1108   running_gcd = CalculateGCDFromCoefficients(source_recurrences, running_gcd);
1109   running_gcd = CalculateGCDFromCoefficients(dest_recurrences, running_gcd);
1110 
1111   return delta % running_gcd != 0;
1112 }
1113 
1114 using PartitionedSubscripts =
1115     std::vector<std::set<std::pair<Instruction*, Instruction*>>>;
PartitionSubscripts(const std::vector<Instruction * > & source_subscripts,const std::vector<Instruction * > & destination_subscripts)1116 PartitionedSubscripts LoopDependenceAnalysis::PartitionSubscripts(
1117     const std::vector<Instruction*>& source_subscripts,
1118     const std::vector<Instruction*>& destination_subscripts) {
1119   PartitionedSubscripts partitions{};
1120 
1121   auto num_subscripts = source_subscripts.size();
1122 
1123   // Create initial partitions with one subscript pair per partition.
1124   for (size_t i = 0; i < num_subscripts; ++i) {
1125     partitions.push_back({{source_subscripts[i], destination_subscripts[i]}});
1126   }
1127 
1128   // Iterate over the loops to create all partitions
1129   for (auto loop : loops_) {
1130     int64_t k = -1;
1131 
1132     for (size_t j = 0; j < partitions.size(); ++j) {
1133       auto& current_partition = partitions[j];
1134 
1135       // Does |loop| appear in |current_partition|
1136       auto it = std::find_if(
1137           current_partition.begin(), current_partition.end(),
1138           [loop,
1139            this](const std::pair<Instruction*, Instruction*>& elem) -> bool {
1140             auto source_recurrences =
1141                 scalar_evolution_.AnalyzeInstruction(std::get<0>(elem))
1142                     ->CollectRecurrentNodes();
1143             auto destination_recurrences =
1144                 scalar_evolution_.AnalyzeInstruction(std::get<1>(elem))
1145                     ->CollectRecurrentNodes();
1146 
1147             source_recurrences.insert(source_recurrences.end(),
1148                                       destination_recurrences.begin(),
1149                                       destination_recurrences.end());
1150 
1151             auto loops_in_pair = CollectLoops(source_recurrences);
1152             auto end_it = loops_in_pair.end();
1153 
1154             return std::find(loops_in_pair.begin(), end_it, loop) != end_it;
1155           });
1156 
1157       auto has_loop = it != current_partition.end();
1158 
1159       if (has_loop) {
1160         if (k == -1) {
1161           k = j;
1162         } else {
1163           // Add |partitions[j]| to |partitions[k]| and discard |partitions[j]|
1164           partitions[static_cast<size_t>(k)].insert(current_partition.begin(),
1165                                                     current_partition.end());
1166           current_partition.clear();
1167         }
1168       }
1169     }
1170   }
1171 
1172   // Remove empty (discarded) partitions
1173   partitions.erase(
1174       std::remove_if(
1175           partitions.begin(), partitions.end(),
1176           [](const std::set<std::pair<Instruction*, Instruction*>>& partition) {
1177             return partition.empty();
1178           }),
1179       partitions.end());
1180 
1181   return partitions;
1182 }
1183 
IntersectConstraints(Constraint * constraint_0,Constraint * constraint_1,const SENode * lower_bound,const SENode * upper_bound)1184 Constraint* LoopDependenceAnalysis::IntersectConstraints(
1185     Constraint* constraint_0, Constraint* constraint_1,
1186     const SENode* lower_bound, const SENode* upper_bound) {
1187   if (constraint_0->AsDependenceNone()) {
1188     return constraint_1;
1189   } else if (constraint_1->AsDependenceNone()) {
1190     return constraint_0;
1191   }
1192 
1193   // Both constraints are distances. Either the same distance or independent.
1194   if (constraint_0->AsDependenceDistance() &&
1195       constraint_1->AsDependenceDistance()) {
1196     auto dist_0 = constraint_0->AsDependenceDistance();
1197     auto dist_1 = constraint_1->AsDependenceDistance();
1198 
1199     if (*dist_0->GetDistance() == *dist_1->GetDistance()) {
1200       return constraint_0;
1201     } else {
1202       return make_constraint<DependenceEmpty>();
1203     }
1204   }
1205 
1206   // Both constraints are points. Either the same point or independent.
1207   if (constraint_0->AsDependencePoint() && constraint_1->AsDependencePoint()) {
1208     auto point_0 = constraint_0->AsDependencePoint();
1209     auto point_1 = constraint_1->AsDependencePoint();
1210 
1211     if (*point_0->GetSource() == *point_1->GetSource() &&
1212         *point_0->GetDestination() == *point_1->GetDestination()) {
1213       return constraint_0;
1214     } else {
1215       return make_constraint<DependenceEmpty>();
1216     }
1217   }
1218 
1219   // Both constraints are lines/distances.
1220   if ((constraint_0->AsDependenceDistance() ||
1221        constraint_0->AsDependenceLine()) &&
1222       (constraint_1->AsDependenceDistance() ||
1223        constraint_1->AsDependenceLine())) {
1224     auto is_distance_0 = constraint_0->AsDependenceDistance() != nullptr;
1225     auto is_distance_1 = constraint_1->AsDependenceDistance() != nullptr;
1226 
1227     auto a0 = is_distance_0 ? scalar_evolution_.CreateConstant(1)
1228                             : constraint_0->AsDependenceLine()->GetA();
1229     auto b0 = is_distance_0 ? scalar_evolution_.CreateConstant(-1)
1230                             : constraint_0->AsDependenceLine()->GetB();
1231     auto c0 =
1232         is_distance_0
1233             ? scalar_evolution_.SimplifyExpression(
1234                   scalar_evolution_.CreateNegation(
1235                       constraint_0->AsDependenceDistance()->GetDistance()))
1236             : constraint_0->AsDependenceLine()->GetC();
1237 
1238     auto a1 = is_distance_1 ? scalar_evolution_.CreateConstant(1)
1239                             : constraint_1->AsDependenceLine()->GetA();
1240     auto b1 = is_distance_1 ? scalar_evolution_.CreateConstant(-1)
1241                             : constraint_1->AsDependenceLine()->GetB();
1242     auto c1 =
1243         is_distance_1
1244             ? scalar_evolution_.SimplifyExpression(
1245                   scalar_evolution_.CreateNegation(
1246                       constraint_1->AsDependenceDistance()->GetDistance()))
1247             : constraint_1->AsDependenceLine()->GetC();
1248 
1249     if (a0->AsSEConstantNode() && b0->AsSEConstantNode() &&
1250         c0->AsSEConstantNode() && a1->AsSEConstantNode() &&
1251         b1->AsSEConstantNode() && c1->AsSEConstantNode()) {
1252       auto constant_a0 = a0->AsSEConstantNode()->FoldToSingleValue();
1253       auto constant_b0 = b0->AsSEConstantNode()->FoldToSingleValue();
1254       auto constant_c0 = c0->AsSEConstantNode()->FoldToSingleValue();
1255 
1256       auto constant_a1 = a1->AsSEConstantNode()->FoldToSingleValue();
1257       auto constant_b1 = b1->AsSEConstantNode()->FoldToSingleValue();
1258       auto constant_c1 = c1->AsSEConstantNode()->FoldToSingleValue();
1259 
1260       // a & b can't both be zero, otherwise it wouldn't be line.
1261       if (NormalizeAndCompareFractions(constant_a0, constant_b0, constant_a1,
1262                                        constant_b1)) {
1263         // Slopes are equal, either parallel lines or the same line.
1264 
1265         if (constant_b0 == 0 && constant_b1 == 0) {
1266           if (NormalizeAndCompareFractions(constant_c0, constant_a0,
1267                                            constant_c1, constant_a1)) {
1268             return constraint_0;
1269           }
1270 
1271           return make_constraint<DependenceEmpty>();
1272         } else if (NormalizeAndCompareFractions(constant_c0, constant_b0,
1273                                                 constant_c1, constant_b1)) {
1274           // Same line.
1275           return constraint_0;
1276         } else {
1277           // Parallel lines can't intersect, report independence.
1278           return make_constraint<DependenceEmpty>();
1279         }
1280 
1281       } else {
1282         // Lines are not parallel, therefore, they must intersect.
1283 
1284         // Calculate intersection.
1285         if (upper_bound->AsSEConstantNode() &&
1286             lower_bound->AsSEConstantNode()) {
1287           auto constant_lower_bound =
1288               lower_bound->AsSEConstantNode()->FoldToSingleValue();
1289           auto constant_upper_bound =
1290               upper_bound->AsSEConstantNode()->FoldToSingleValue();
1291 
1292           auto up = constant_b1 * constant_c0 - constant_b0 * constant_c1;
1293           // Both b or both a can't be 0, so down is never 0
1294           // otherwise would have entered the parallel line section.
1295           auto down = constant_b1 * constant_a0 - constant_b0 * constant_a1;
1296 
1297           auto x_coord = up / down;
1298 
1299           int64_t y_coord = 0;
1300           int64_t arg1 = 0;
1301           int64_t const_b_to_use = 0;
1302 
1303           if (constant_b1 != 0) {
1304             arg1 = constant_c1 - constant_a1 * x_coord;
1305             y_coord = arg1 / constant_b1;
1306             const_b_to_use = constant_b1;
1307           } else if (constant_b0 != 0) {
1308             arg1 = constant_c0 - constant_a0 * x_coord;
1309             y_coord = arg1 / constant_b0;
1310             const_b_to_use = constant_b0;
1311           }
1312 
1313           if (up % down == 0 &&
1314               arg1 % const_b_to_use == 0 &&  // Coordinates are integers.
1315               constant_lower_bound <=
1316                   x_coord &&  // x_coord is within loop bounds.
1317               x_coord <= constant_upper_bound &&
1318               constant_lower_bound <=
1319                   y_coord &&  // y_coord is within loop bounds.
1320               y_coord <= constant_upper_bound) {
1321             // Lines intersect at integer coordinates.
1322             return make_constraint<DependencePoint>(
1323                 scalar_evolution_.CreateConstant(x_coord),
1324                 scalar_evolution_.CreateConstant(y_coord),
1325                 constraint_0->GetLoop());
1326 
1327           } else {
1328             return make_constraint<DependenceEmpty>();
1329           }
1330 
1331         } else {
1332           // Not constants, bail out.
1333           return make_constraint<DependenceNone>();
1334         }
1335       }
1336 
1337     } else {
1338       // Not constants, bail out.
1339       return make_constraint<DependenceNone>();
1340     }
1341   }
1342 
1343   // One constraint is a line/distance and the other is a point.
1344   if ((constraint_0->AsDependencePoint() &&
1345        (constraint_1->AsDependenceLine() ||
1346         constraint_1->AsDependenceDistance())) ||
1347       (constraint_1->AsDependencePoint() &&
1348        (constraint_0->AsDependenceLine() ||
1349         constraint_0->AsDependenceDistance()))) {
1350     auto point_0 = constraint_0->AsDependencePoint() != nullptr;
1351 
1352     auto point = point_0 ? constraint_0->AsDependencePoint()
1353                          : constraint_1->AsDependencePoint();
1354 
1355     auto line_or_distance = point_0 ? constraint_1 : constraint_0;
1356 
1357     auto is_distance = line_or_distance->AsDependenceDistance() != nullptr;
1358 
1359     auto a = is_distance ? scalar_evolution_.CreateConstant(1)
1360                          : line_or_distance->AsDependenceLine()->GetA();
1361     auto b = is_distance ? scalar_evolution_.CreateConstant(-1)
1362                          : line_or_distance->AsDependenceLine()->GetB();
1363     auto c =
1364         is_distance
1365             ? scalar_evolution_.SimplifyExpression(
1366                   scalar_evolution_.CreateNegation(
1367                       line_or_distance->AsDependenceDistance()->GetDistance()))
1368             : line_or_distance->AsDependenceLine()->GetC();
1369 
1370     auto x = point->GetSource();
1371     auto y = point->GetDestination();
1372 
1373     if (a->AsSEConstantNode() && b->AsSEConstantNode() &&
1374         c->AsSEConstantNode() && x->AsSEConstantNode() &&
1375         y->AsSEConstantNode()) {
1376       auto constant_a = a->AsSEConstantNode()->FoldToSingleValue();
1377       auto constant_b = b->AsSEConstantNode()->FoldToSingleValue();
1378       auto constant_c = c->AsSEConstantNode()->FoldToSingleValue();
1379 
1380       auto constant_x = x->AsSEConstantNode()->FoldToSingleValue();
1381       auto constant_y = y->AsSEConstantNode()->FoldToSingleValue();
1382 
1383       auto left_hand_side = constant_a * constant_x + constant_b * constant_y;
1384 
1385       if (left_hand_side == constant_c) {
1386         // Point is on line, return point
1387         return point_0 ? constraint_0 : constraint_1;
1388       } else {
1389         // Point not on line, report independence (empty constraint).
1390         return make_constraint<DependenceEmpty>();
1391       }
1392 
1393     } else {
1394       // Not constants, bail out.
1395       return make_constraint<DependenceNone>();
1396     }
1397   }
1398 
1399   return nullptr;
1400 }
1401 
1402 // Propagate constraints function as described in section 5 of Practical
1403 // Dependence Testing, Goff, Kennedy, Tseng, 1991.
PropagateConstraints(const SubscriptPair & subscript_pair,const std::vector<Constraint * > & constraints)1404 SubscriptPair LoopDependenceAnalysis::PropagateConstraints(
1405     const SubscriptPair& subscript_pair,
1406     const std::vector<Constraint*>& constraints) {
1407   SENode* new_first = subscript_pair.first;
1408   SENode* new_second = subscript_pair.second;
1409 
1410   for (auto& constraint : constraints) {
1411     // In the paper this is a[k]. We're extracting the coefficient ('a') of a
1412     // recurrent expression with respect to the loop 'k'.
1413     SENode* coefficient_of_recurrent =
1414         scalar_evolution_.GetCoefficientFromRecurrentTerm(
1415             new_first, constraint->GetLoop());
1416 
1417     // In the paper this is a'[k].
1418     SENode* coefficient_of_recurrent_prime =
1419         scalar_evolution_.GetCoefficientFromRecurrentTerm(
1420             new_second, constraint->GetLoop());
1421 
1422     if (constraint->GetType() == Constraint::Distance) {
1423       DependenceDistance* as_distance = constraint->AsDependenceDistance();
1424 
1425       // In the paper this is a[k]*d
1426       SENode* rhs = scalar_evolution_.CreateMultiplyNode(
1427           coefficient_of_recurrent, as_distance->GetDistance());
1428 
1429       // In the paper this is a[k] <- 0
1430       SENode* zeroed_coefficient =
1431           scalar_evolution_.BuildGraphWithoutRecurrentTerm(
1432               new_first, constraint->GetLoop());
1433 
1434       // In the paper this is e <- e - a[k]*d.
1435       new_first = scalar_evolution_.CreateSubtraction(zeroed_coefficient, rhs);
1436       new_first = scalar_evolution_.SimplifyExpression(new_first);
1437 
1438       // In the paper this is a'[k] - a[k].
1439       SENode* new_child = scalar_evolution_.SimplifyExpression(
1440           scalar_evolution_.CreateSubtraction(coefficient_of_recurrent_prime,
1441                                               coefficient_of_recurrent));
1442 
1443       // In the paper this is a'[k]'i[k].
1444       SERecurrentNode* prime_recurrent =
1445           scalar_evolution_.GetRecurrentTerm(new_second, constraint->GetLoop());
1446 
1447       if (!prime_recurrent) continue;
1448 
1449       // As we hash the nodes we need to create a new node when we update a
1450       // child.
1451       SENode* new_recurrent = scalar_evolution_.CreateRecurrentExpression(
1452           constraint->GetLoop(), prime_recurrent->GetOffset(), new_child);
1453       // In the paper this is a'[k] <- a'[k] - a[k].
1454       new_second = scalar_evolution_.UpdateChildNode(
1455           new_second, prime_recurrent, new_recurrent);
1456     }
1457   }
1458 
1459   new_second = scalar_evolution_.SimplifyExpression(new_second);
1460   return std::make_pair(new_first, new_second);
1461 }
1462 
DeltaTest(const std::vector<SubscriptPair> & coupled_subscripts,DistanceVector * dv_entry)1463 bool LoopDependenceAnalysis::DeltaTest(
1464     const std::vector<SubscriptPair>& coupled_subscripts,
1465     DistanceVector* dv_entry) {
1466   std::vector<Constraint*> constraints(loops_.size());
1467 
1468   std::vector<bool> loop_appeared(loops_.size());
1469 
1470   std::generate(std::begin(constraints), std::end(constraints),
1471                 [this]() { return make_constraint<DependenceNone>(); });
1472 
1473   // Separate SIV and MIV subscripts
1474   std::vector<SubscriptPair> siv_subscripts{};
1475   std::vector<SubscriptPair> miv_subscripts{};
1476 
1477   for (const auto& subscript_pair : coupled_subscripts) {
1478     if (IsSIV(subscript_pair)) {
1479       siv_subscripts.push_back(subscript_pair);
1480     } else {
1481       miv_subscripts.push_back(subscript_pair);
1482     }
1483   }
1484 
1485   // Delta Test
1486   while (!siv_subscripts.empty()) {
1487     std::vector<bool> results(siv_subscripts.size());
1488 
1489     std::vector<DistanceVector> current_distances(
1490         siv_subscripts.size(), DistanceVector(loops_.size()));
1491 
1492     // Apply SIV test to all SIV subscripts, report independence if any of them
1493     // is independent
1494     std::transform(
1495         std::begin(siv_subscripts), std::end(siv_subscripts),
1496         std::begin(current_distances), std::begin(results),
1497         [this](SubscriptPair& p, DistanceVector& d) { return SIVTest(p, &d); });
1498 
1499     if (std::accumulate(std::begin(results), std::end(results), false,
1500                         std::logical_or<bool>{})) {
1501       return true;
1502     }
1503 
1504     // Derive new constraint vector.
1505     std::vector<std::pair<Constraint*, size_t>> all_new_constrants{};
1506 
1507     for (size_t i = 0; i < siv_subscripts.size(); ++i) {
1508       auto loop = GetLoopForSubscriptPair(siv_subscripts[i]);
1509 
1510       auto loop_id =
1511           std::distance(std::begin(loops_),
1512                         std::find(std::begin(loops_), std::end(loops_), loop));
1513 
1514       loop_appeared[loop_id] = true;
1515       auto distance_entry = current_distances[i].GetEntries()[loop_id];
1516 
1517       if (distance_entry.dependence_information ==
1518           DistanceEntry::DependenceInformation::DISTANCE) {
1519         // Construct a DependenceDistance.
1520         auto node = scalar_evolution_.CreateConstant(distance_entry.distance);
1521 
1522         all_new_constrants.push_back(
1523             {make_constraint<DependenceDistance>(node, loop), loop_id});
1524       } else {
1525         // Construct a DependenceLine.
1526         const auto& subscript_pair = siv_subscripts[i];
1527         SENode* source_node = std::get<0>(subscript_pair);
1528         SENode* destination_node = std::get<1>(subscript_pair);
1529 
1530         int64_t source_induction_count = CountInductionVariables(source_node);
1531         int64_t destination_induction_count =
1532             CountInductionVariables(destination_node);
1533 
1534         SENode* a = nullptr;
1535         SENode* b = nullptr;
1536         SENode* c = nullptr;
1537 
1538         if (destination_induction_count != 0) {
1539           a = destination_node->AsSERecurrentNode()->GetCoefficient();
1540           c = scalar_evolution_.CreateNegation(
1541               destination_node->AsSERecurrentNode()->GetOffset());
1542         } else {
1543           a = scalar_evolution_.CreateConstant(0);
1544           c = scalar_evolution_.CreateNegation(destination_node);
1545         }
1546 
1547         if (source_induction_count != 0) {
1548           b = scalar_evolution_.CreateNegation(
1549               source_node->AsSERecurrentNode()->GetCoefficient());
1550           c = scalar_evolution_.CreateAddNode(
1551               c, source_node->AsSERecurrentNode()->GetOffset());
1552         } else {
1553           b = scalar_evolution_.CreateConstant(0);
1554           c = scalar_evolution_.CreateAddNode(c, source_node);
1555         }
1556 
1557         a = scalar_evolution_.SimplifyExpression(a);
1558         b = scalar_evolution_.SimplifyExpression(b);
1559         c = scalar_evolution_.SimplifyExpression(c);
1560 
1561         all_new_constrants.push_back(
1562             {make_constraint<DependenceLine>(a, b, c, loop), loop_id});
1563       }
1564     }
1565 
1566     // Calculate the intersection between the new and existing constraints.
1567     std::vector<Constraint*> intersection = constraints;
1568     for (const auto& constraint_to_intersect : all_new_constrants) {
1569       auto loop_id = std::get<1>(constraint_to_intersect);
1570       auto loop = loops_[loop_id];
1571       intersection[loop_id] = IntersectConstraints(
1572           intersection[loop_id], std::get<0>(constraint_to_intersect),
1573           GetLowerBound(loop), GetUpperBound(loop));
1574     }
1575 
1576     // Report independence if an empty constraint (DependenceEmpty) is found.
1577     auto first_empty =
1578         std::find_if(std::begin(intersection), std::end(intersection),
1579                      [](Constraint* constraint) {
1580                        return constraint->AsDependenceEmpty() != nullptr;
1581                      });
1582     if (first_empty != std::end(intersection)) {
1583       return true;
1584     }
1585     std::vector<SubscriptPair> new_siv_subscripts{};
1586     std::vector<SubscriptPair> new_miv_subscripts{};
1587 
1588     auto equal =
1589         std::equal(std::begin(constraints), std::end(constraints),
1590                    std::begin(intersection),
1591                    [](Constraint* a, Constraint* b) { return *a == *b; });
1592 
1593     // If any constraints have changed, propagate them into the rest of the
1594     // subscripts possibly creating new ZIV/SIV subscripts.
1595     if (!equal) {
1596       std::vector<SubscriptPair> new_subscripts(miv_subscripts.size());
1597 
1598       // Propagate constraints into MIV subscripts
1599       std::transform(std::begin(miv_subscripts), std::end(miv_subscripts),
1600                      std::begin(new_subscripts),
1601                      [this, &intersection](SubscriptPair& subscript_pair) {
1602                        return PropagateConstraints(subscript_pair,
1603                                                    intersection);
1604                      });
1605 
1606       // If a ZIV subscript is returned, apply test, otherwise, update untested
1607       // subscripts.
1608       for (auto& subscript : new_subscripts) {
1609         if (IsZIV(subscript) && ZIVTest(subscript)) {
1610           return true;
1611         } else if (IsSIV(subscript)) {
1612           new_siv_subscripts.push_back(subscript);
1613         } else {
1614           new_miv_subscripts.push_back(subscript);
1615         }
1616       }
1617     }
1618 
1619     // Set new constraints and subscripts to test.
1620     std::swap(siv_subscripts, new_siv_subscripts);
1621     std::swap(miv_subscripts, new_miv_subscripts);
1622     std::swap(constraints, intersection);
1623   }
1624 
1625   // Create the dependence vector from the constraints.
1626   for (size_t i = 0; i < loops_.size(); ++i) {
1627     // Don't touch entries for loops that weren't tested.
1628     if (loop_appeared[i]) {
1629       auto current_constraint = constraints[i];
1630       auto& current_distance_entry = (*dv_entry).GetEntries()[i];
1631 
1632       if (auto dependence_distance =
1633               current_constraint->AsDependenceDistance()) {
1634         if (auto constant_node =
1635                 dependence_distance->GetDistance()->AsSEConstantNode()) {
1636           current_distance_entry.dependence_information =
1637               DistanceEntry::DependenceInformation::DISTANCE;
1638 
1639           current_distance_entry.distance = constant_node->FoldToSingleValue();
1640           if (current_distance_entry.distance == 0) {
1641             current_distance_entry.direction = DistanceEntry::Directions::EQ;
1642           } else if (current_distance_entry.distance < 0) {
1643             current_distance_entry.direction = DistanceEntry::Directions::GT;
1644           } else {
1645             current_distance_entry.direction = DistanceEntry::Directions::LT;
1646           }
1647         }
1648       } else if (auto dependence_point =
1649                      current_constraint->AsDependencePoint()) {
1650         auto source = dependence_point->GetSource();
1651         auto destination = dependence_point->GetDestination();
1652 
1653         if (source->AsSEConstantNode() && destination->AsSEConstantNode()) {
1654           current_distance_entry = DistanceEntry(
1655               source->AsSEConstantNode()->FoldToSingleValue(),
1656               destination->AsSEConstantNode()->FoldToSingleValue());
1657         }
1658       }
1659     }
1660   }
1661 
1662   // Test any remaining MIV subscripts and report independence if found.
1663   std::vector<bool> results(miv_subscripts.size());
1664 
1665   std::transform(std::begin(miv_subscripts), std::end(miv_subscripts),
1666                  std::begin(results),
1667                  [this](const SubscriptPair& p) { return GCDMIVTest(p); });
1668 
1669   return std::accumulate(std::begin(results), std::end(results), false,
1670                          std::logical_or<bool>{});
1671 }
1672 
1673 }  // namespace opt
1674 }  // namespace spvtools
1675