27 def GenerateZipLanes(emitter, registers, zip_lanes, input_address, stride):  argument
50       emitter.EmitAdd(address_register, last_address_register, stride)
64 def GenerateClearAggregators(emitter, lanes):  argument
66     emitter.EmitVMov('i16', lane.aggregator, emitter.ImmediateConstant(0))
69 def GenerateLoadAggregateStore(emitter, lanes, output_address, alignment):  argument
71   emitter.EmitNewline()
72   emitter.EmitComment('Load Aggregate Store.')
75     emitter.EmitVLoad(
77         emitter.DereferenceIncrement(lane.input_address, alignment))
81     emitter.EmitVAddw('u8', lane.aggregator, lane.aggregator, lane.load)
84   emitter.EmitVStoreA('1.8', store_registers,
85                       emitter.DereferenceIncrement(output_address, 64))
88 def GenerateLeftoverLoadAggregateStore(emitter, leftovers, lanes,  argument
91   emitter.EmitNewline()
92   emitter.EmitComment('Leftover Load Aggregate Store.')
96     emitter.EmitVMov('i8', lane.load, emitter.ImmediateConstant(0))
101       emitter.EmitVLoad('1.8', emitter.Lane(lane.load, 0),
102                         emitter.Dereference(lane.input_address, None))
106       emitter.EmitVLoad('1.16', emitter.Lane(lane.load, 0),
107                         emitter.Dereference(lane.input_address, None))
111       emitter.EmitVLoad('1.16', emitter.Lane(lane.load, 0),
112                         emitter.DereferenceIncrement(lane.input_address, None))
115       emitter.EmitVLoad('1.8', emitter.Lane(lane.load, 2),
116                         emitter.Dereference(lane.input_address, None))
120       emitter.EmitVLoad('1.32', emitter.Lane(lane.load, 0),
121                         emitter.Dereference(lane.input_address, None))
125       emitter.EmitVLoad('1.32', emitter.Lane(lane.load, 0),
126                         emitter.DereferenceIncrement(lane.input_address, None))
129       emitter.EmitVLoad('1.8', emitter.Lane(lane.load, 4),
130                         emitter.Dereference(lane.input_address, None))
134       emitter.EmitVLoad('1.32', emitter.Lane(lane.load, 0),
135                         emitter.DereferenceIncrement(lane.input_address, None))
138       emitter.EmitVLoad('1.16', emitter.Lane(lane.load, 2),
139                         emitter.Dereference(lane.input_address, None))
143       emitter.EmitVLoad('1.32', emitter.Lane(lane.load, 0),
144                         emitter.DereferenceIncrement(lane.input_address, None))
147       emitter.EmitVLoad('1.16', emitter.Lane(lane.load, 2),
148                         emitter.DereferenceIncrement(lane.input_address, None))
151       emitter.EmitVLoad('1.8', emitter.Lane(lane.load, 6),
152                         emitter.Dereference(lane.input_address, None))
159     emitter.EmitVAddw('u8', lane.aggregator, lane.aggregator, lane.load)
163   emitter.EmitVStoreA('1.8', store_registers,
164                       emitter.DereferenceIncrement(output_address, 64))
167 def GenerateAggregatorReduction(emitter, registers, lanes, output_address,  argument
170   emitter.EmitNewline()
171   emitter.EmitComment('Aggregator Reduction.')
174   emitter.EmitVMov('32', emitter.Lane(multiplier, 0), multiplicative_offset)
176   emitter.EmitVDup('32', offset, additive_offset)
180     emitter.EmitVPaddl('u16', lane.aggregator, lane.aggregator)
185     emitter.EmitVPadd('u32', lane_temp, registers.Low(lane.aggregator),
193     emitter.EmitVPadd('u32', low, lane_temps[0], lane_temps[0])
195     emitter.EmitVPadd('u32', low, lane_temps[0], lane_temps[1])
197     emitter.EmitVPadd('u32', low, lane_temps[0], lane_temps[1])
198     emitter.EmitVPadd('u32', high, lane_temps[2], lane_temps[2])
200     emitter.EmitVPadd('u32', low, lane_temps[0], lane_temps[1])
201     emitter.EmitVPadd('u32', high, lane_temps[2], lane_temps[3])
206   emitter.EmitVMul('i32', temp, temp, emitter.Lane(multiplier, 0))
207   emitter.EmitVAdd('i32', temp, temp, offset)
210     emitter.EmitVStore('1.32', emitter.Lane(low, 0),
211                        emitter.Dereference(output_address, None))
213     emitter.EmitVStore('1.32', low, emitter.Dereference(output_address, 64))
215     emitter.EmitVStore('1.32', low,
216                        emitter.DereferenceIncrement(output_address, 64))
217     emitter.EmitVStore('1.32', emitter.Lane(high, 0),
218                        emitter.Dereference(output_address, None))
220     emitter.EmitVStoreA('1.32', [low, high],
221                         emitter.DereferenceIncrement(output_address, 64))
224 def GenerateZipNx8(emitter, zip_lanes, leftovers, aligned):  argument
233   emitter.EmitFunctionBeginA(
238   emitter.EmitAssert('count %% 8 == %d' % leftovers)
239   emitter.EmitAssert('count <= 2048')
240   emitter.EmitAssert('count >= 8')
241   emitter.EmitAssert('reinterpret_cast<std::uintptr_t>(destination) % 8 == 0')
243     emitter.EmitAssert('reinterpret_cast<std::uintptr_t>(source) % 8 == 0')
245       emitter.EmitAssert('stride % 8 == 0')
246   emitter.EmitAsmBegin()
253   lanes = GenerateZipLanes(emitter, registers, zip_lanes,
258     emitter.EmitSub(count, count, emitter.ImmediateConstant(leftovers))
260   GenerateClearAggregators(emitter, lanes)
262   emitter.EmitNewline()
263   emitter.EmitNumericalLabel(1)
264   emitter.EmitSubs(count, count, emitter.ImmediateConstant(8))
266   GenerateLoadAggregateStore(emitter, lanes, output_address, 64 if aligned else
269   emitter.EmitNewline()
270   emitter.EmitBneBack(1)
273     GenerateLeftoverLoadAggregateStore(emitter, leftovers, lanes,
276   GenerateAggregatorReduction(emitter, registers, lanes, output_address,
280   emitter.EmitAsmEnd(registers.MappedParameters(), [],
282   emitter.EmitFunctionEnd()
285 def GenerateFunctions(emitter):  argument
289         GenerateZipNx8(emitter, lanes, leftovers, aligned)
290         emitter.EmitNewline()