1 /*
2 * Copyright (C) 2011 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26 #include "config.h"
27 #include "DFGJITCodeGenerator.h"
28
29 #if ENABLE(DFG_JIT)
30
31 #include "DFGNonSpeculativeJIT.h"
32 #include "DFGSpeculativeJIT.h"
33 #include "LinkBuffer.h"
34
35 namespace JSC { namespace DFG {
36
fillInteger(NodeIndex nodeIndex,DataFormat & returnFormat)37 GPRReg JITCodeGenerator::fillInteger(NodeIndex nodeIndex, DataFormat& returnFormat)
38 {
39 Node& node = m_jit.graph()[nodeIndex];
40 VirtualRegister virtualRegister = node.virtualRegister;
41 GenerationInfo& info = m_generationInfo[virtualRegister];
42
43 if (info.registerFormat() == DataFormatNone) {
44 GPRReg gpr = allocate();
45 JITCompiler::RegisterID reg = JITCompiler::gprToRegisterID(gpr);
46
47 if (node.isConstant()) {
48 m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
49 if (isInt32Constant(nodeIndex)) {
50 m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(nodeIndex)), reg);
51 info.fillInteger(gpr);
52 returnFormat = DataFormatInteger;
53 return gpr;
54 }
55 if (isDoubleConstant(nodeIndex)) {
56 JSValue jsValue = jsNumber(valueOfDoubleConstant(nodeIndex));
57 m_jit.move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), reg);
58 } else {
59 ASSERT(isJSConstant(nodeIndex));
60 JSValue jsValue = valueOfJSConstant(nodeIndex);
61 m_jit.move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), reg);
62 }
63 } else {
64 ASSERT(info.spillFormat() == DataFormatJS || info.spillFormat() == DataFormatJSInteger);
65 m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
66 m_jit.loadPtr(JITCompiler::addressFor(virtualRegister), reg);
67 }
68
69 // Since we statically know that we're filling an integer, and values
70 // in the RegisterFile are boxed, this must be DataFormatJSInteger.
71 // We will check this with a jitAssert below.
72 info.fillJSValue(gpr, DataFormatJSInteger);
73 unlock(gpr);
74 }
75
76 switch (info.registerFormat()) {
77 case DataFormatNone:
78 // Should have filled, above.
79 case DataFormatJSDouble:
80 case DataFormatDouble:
81 case DataFormatJS:
82 case DataFormatCell:
83 case DataFormatJSCell:
84 // Should only be calling this function if we know this operand to be integer.
85 ASSERT_NOT_REACHED();
86
87 case DataFormatJSInteger: {
88 GPRReg gpr = info.gpr();
89 m_gprs.lock(gpr);
90 m_jit.jitAssertIsJSInt32(gpr);
91 returnFormat = DataFormatJSInteger;
92 return gpr;
93 }
94
95 case DataFormatInteger: {
96 GPRReg gpr = info.gpr();
97 m_gprs.lock(gpr);
98 m_jit.jitAssertIsInt32(gpr);
99 returnFormat = DataFormatInteger;
100 return gpr;
101 }
102 }
103
104 ASSERT_NOT_REACHED();
105 return InvalidGPRReg;
106 }
107
fillDouble(NodeIndex nodeIndex)108 FPRReg JITCodeGenerator::fillDouble(NodeIndex nodeIndex)
109 {
110 Node& node = m_jit.graph()[nodeIndex];
111 VirtualRegister virtualRegister = node.virtualRegister;
112 GenerationInfo& info = m_generationInfo[virtualRegister];
113
114 if (info.registerFormat() == DataFormatNone) {
115 GPRReg gpr = allocate();
116 JITCompiler::RegisterID reg = JITCompiler::gprToRegisterID(gpr);
117
118 if (node.isConstant()) {
119 if (isInt32Constant(nodeIndex)) {
120 // FIXME: should not be reachable?
121 m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(nodeIndex)), reg);
122 m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
123 info.fillInteger(gpr);
124 unlock(gpr);
125 } else if (isDoubleConstant(nodeIndex)) {
126 FPRReg fpr = fprAllocate();
127 m_jit.move(MacroAssembler::ImmPtr(reinterpret_cast<void*>(reinterpretDoubleToIntptr(valueOfDoubleConstant(nodeIndex)))), reg);
128 m_jit.movePtrToDouble(reg, JITCompiler::fprToRegisterID(fpr));
129 unlock(gpr);
130
131 m_fprs.retain(fpr, virtualRegister, SpillOrderDouble);
132 info.fillDouble(fpr);
133 return fpr;
134 } else {
135 // FIXME: should not be reachable?
136 ASSERT(isJSConstant(nodeIndex));
137 JSValue jsValue = valueOfJSConstant(nodeIndex);
138 m_jit.move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), reg);
139 m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
140 info.fillJSValue(gpr, DataFormatJS);
141 unlock(gpr);
142 }
143 } else {
144 DataFormat spillFormat = info.spillFormat();
145 ASSERT(spillFormat & DataFormatJS);
146 m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
147 m_jit.loadPtr(JITCompiler::addressFor(virtualRegister), reg);
148 info.fillJSValue(gpr, m_isSpeculative ? spillFormat : DataFormatJS);
149 unlock(gpr);
150 }
151 }
152
153 switch (info.registerFormat()) {
154 case DataFormatNone:
155 // Should have filled, above.
156 case DataFormatCell:
157 case DataFormatJSCell:
158 // Should only be calling this function if we know this operand to be numeric.
159 ASSERT_NOT_REACHED();
160
161 case DataFormatJS: {
162 GPRReg jsValueGpr = info.gpr();
163 m_gprs.lock(jsValueGpr);
164 FPRReg fpr = fprAllocate();
165 GPRReg tempGpr = allocate(); // FIXME: can we skip this allocation on the last use of the virtual register?
166
167 JITCompiler::RegisterID jsValueReg = JITCompiler::gprToRegisterID(jsValueGpr);
168 JITCompiler::FPRegisterID fpReg = JITCompiler::fprToRegisterID(fpr);
169 JITCompiler::RegisterID tempReg = JITCompiler::gprToRegisterID(tempGpr);
170
171 JITCompiler::Jump isInteger = m_jit.branchPtr(MacroAssembler::AboveOrEqual, jsValueReg, JITCompiler::tagTypeNumberRegister);
172
173 m_jit.jitAssertIsJSDouble(jsValueGpr);
174
175 // First, if we get here we have a double encoded as a JSValue
176 m_jit.move(jsValueReg, tempReg);
177 m_jit.addPtr(JITCompiler::tagTypeNumberRegister, tempReg);
178 m_jit.movePtrToDouble(tempReg, fpReg);
179 JITCompiler::Jump hasUnboxedDouble = m_jit.jump();
180
181 // Finally, handle integers.
182 isInteger.link(&m_jit);
183 m_jit.convertInt32ToDouble(jsValueReg, fpReg);
184 hasUnboxedDouble.link(&m_jit);
185
186 m_gprs.release(jsValueGpr);
187 m_gprs.unlock(jsValueGpr);
188 m_gprs.unlock(tempGpr);
189 m_fprs.retain(fpr, virtualRegister, SpillOrderDouble);
190 info.fillDouble(fpr);
191 return fpr;
192 }
193
194 case DataFormatJSInteger:
195 case DataFormatInteger: {
196 FPRReg fpr = fprAllocate();
197 GPRReg gpr = info.gpr();
198 m_gprs.lock(gpr);
199 JITCompiler::RegisterID reg = JITCompiler::gprToRegisterID(gpr);
200 JITCompiler::FPRegisterID fpReg = JITCompiler::fprToRegisterID(fpr);
201
202 m_jit.convertInt32ToDouble(reg, fpReg);
203
204 m_gprs.release(gpr);
205 m_gprs.unlock(gpr);
206 m_fprs.retain(fpr, virtualRegister, SpillOrderDouble);
207 info.fillDouble(fpr);
208 return fpr;
209 }
210
211 // Unbox the double
212 case DataFormatJSDouble: {
213 GPRReg gpr = info.gpr();
214 FPRReg fpr = unboxDouble(gpr);
215
216 m_gprs.release(gpr);
217 m_fprs.retain(fpr, virtualRegister, SpillOrderDouble);
218
219 info.fillDouble(fpr);
220 return fpr;
221 }
222
223 case DataFormatDouble: {
224 FPRReg fpr = info.fpr();
225 m_fprs.lock(fpr);
226 return fpr;
227 }
228 }
229
230 ASSERT_NOT_REACHED();
231 return InvalidFPRReg;
232 }
233
fillJSValue(NodeIndex nodeIndex)234 GPRReg JITCodeGenerator::fillJSValue(NodeIndex nodeIndex)
235 {
236 Node& node = m_jit.graph()[nodeIndex];
237 VirtualRegister virtualRegister = node.virtualRegister;
238 GenerationInfo& info = m_generationInfo[virtualRegister];
239
240 switch (info.registerFormat()) {
241 case DataFormatNone: {
242 GPRReg gpr = allocate();
243 JITCompiler::RegisterID reg = JITCompiler::gprToRegisterID(gpr);
244
245 if (node.isConstant()) {
246 if (isInt32Constant(nodeIndex)) {
247 info.fillJSValue(gpr, DataFormatJSInteger);
248 JSValue jsValue = jsNumber(valueOfInt32Constant(nodeIndex));
249 m_jit.move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), reg);
250 } else if (isDoubleConstant(nodeIndex)) {
251 info.fillJSValue(gpr, DataFormatJSDouble);
252 JSValue jsValue(JSValue::EncodeAsDouble, valueOfDoubleConstant(nodeIndex));
253 m_jit.move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), reg);
254 } else {
255 ASSERT(isJSConstant(nodeIndex));
256 JSValue jsValue = valueOfJSConstant(nodeIndex);
257 m_jit.move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), reg);
258 info.fillJSValue(gpr, DataFormatJS);
259 }
260
261 m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
262 } else {
263 DataFormat spillFormat = info.spillFormat();
264 ASSERT(spillFormat & DataFormatJS);
265 m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
266 m_jit.loadPtr(JITCompiler::addressFor(virtualRegister), reg);
267 info.fillJSValue(gpr, m_isSpeculative ? spillFormat : DataFormatJS);
268 }
269 return gpr;
270 }
271
272 case DataFormatInteger: {
273 GPRReg gpr = info.gpr();
274 // If the register has already been locked we need to take a copy.
275 // If not, we'll zero extend in place, so mark on the info that this is now type DataFormatInteger, not DataFormatJSInteger.
276 if (m_gprs.isLocked(gpr)) {
277 GPRReg result = allocate();
278 m_jit.orPtr(JITCompiler::tagTypeNumberRegister, JITCompiler::gprToRegisterID(gpr), JITCompiler::gprToRegisterID(result));
279 return result;
280 }
281 m_gprs.lock(gpr);
282 m_jit.orPtr(JITCompiler::tagTypeNumberRegister, JITCompiler::gprToRegisterID(gpr));
283 info.fillJSValue(gpr, DataFormatJSInteger);
284 return gpr;
285 }
286
287 case DataFormatDouble: {
288 FPRReg fpr = info.fpr();
289 GPRReg gpr = boxDouble(fpr);
290
291 // Update all info
292 info.fillJSValue(gpr, DataFormatJSDouble);
293 m_fprs.release(fpr);
294 m_gprs.retain(gpr, virtualRegister, SpillOrderJS);
295
296 return gpr;
297 }
298
299 case DataFormatCell:
300 // No retag required on JSVALUE64!
301 case DataFormatJS:
302 case DataFormatJSInteger:
303 case DataFormatJSDouble:
304 case DataFormatJSCell: {
305 GPRReg gpr = info.gpr();
306 m_gprs.lock(gpr);
307 return gpr;
308 }
309 }
310
311 ASSERT_NOT_REACHED();
312 return InvalidGPRReg;
313 }
314
useChildren(Node & node)315 void JITCodeGenerator::useChildren(Node& node)
316 {
317 NodeIndex child1 = node.child1;
318 if (child1 == NoNode) {
319 ASSERT(node.child2 == NoNode && node.child3 == NoNode);
320 return;
321 }
322 use(child1);
323
324 NodeIndex child2 = node.child2;
325 if (child2 == NoNode) {
326 ASSERT(node.child3 == NoNode);
327 return;
328 }
329 use(child2);
330
331 NodeIndex child3 = node.child3;
332 if (child3 == NoNode)
333 return;
334 use(child3);
335 }
336
337 #ifndef NDEBUG
dataFormatString(DataFormat format)338 static const char* dataFormatString(DataFormat format)
339 {
340 // These values correspond to the DataFormat enum.
341 const char* strings[] = {
342 "[ ]",
343 "[ i]",
344 "[ d]",
345 "[ c]",
346 "Err!",
347 "Err!",
348 "Err!",
349 "Err!",
350 "[J ]",
351 "[Ji]",
352 "[Jd]",
353 "[Jc]",
354 "Err!",
355 "Err!",
356 "Err!",
357 "Err!",
358 };
359 return strings[format];
360 }
361
dump(const char * label)362 void JITCodeGenerator::dump(const char* label)
363 {
364 if (label)
365 fprintf(stderr, "<%s>\n", label);
366
367 fprintf(stderr, " gprs:\n");
368 m_gprs.dump();
369 fprintf(stderr, " fprs:\n");
370 m_fprs.dump();
371 fprintf(stderr, " VirtualRegisters:\n");
372 for (unsigned i = 0; i < m_generationInfo.size(); ++i) {
373 GenerationInfo& info = m_generationInfo[i];
374 if (info.alive())
375 fprintf(stderr, " % 3d:%s%s\n", i, dataFormatString(info.registerFormat()), dataFormatString(info.spillFormat()));
376 else
377 fprintf(stderr, " % 3d:[__][__]\n", i);
378 }
379 if (label)
380 fprintf(stderr, "</%s>\n", label);
381 }
382 #endif
383
384
385 #if DFG_CONSISTENCY_CHECK
checkConsistency()386 void JITCodeGenerator::checkConsistency()
387 {
388 VirtualRegister grpContents[numberOfGPRs];
389 VirtualRegister frpContents[numberOfFPRs];
390
391 for (unsigned i = 0; i < numberOfGPRs; ++i)
392 grpContents[i] = InvalidVirtualRegister;
393 for (unsigned i = 0; i < numberOfFPRs; ++i)
394 frpContents[i] = InvalidVirtualRegister;
395 for (unsigned i = 0; i < m_generationInfo.size(); ++i) {
396 GenerationInfo& info = m_generationInfo[i];
397 if (!info.alive())
398 continue;
399 switch (info.registerFormat()) {
400 case DataFormatNone:
401 break;
402 case DataFormatInteger:
403 case DataFormatCell:
404 case DataFormatJS:
405 case DataFormatJSInteger:
406 case DataFormatJSDouble:
407 case DataFormatJSCell: {
408 GPRReg gpr = info.gpr();
409 ASSERT(gpr != InvalidGPRReg);
410 grpContents[gpr] = (VirtualRegister)i;
411 break;
412 }
413 case DataFormatDouble: {
414 FPRReg fpr = info.fpr();
415 ASSERT(fpr != InvalidFPRReg);
416 frpContents[fpr] = (VirtualRegister)i;
417 break;
418 }
419 }
420 }
421
422 for (GPRReg i = gpr0; i < numberOfGPRs; next(i)) {
423 if (m_gprs.isLocked(i) || m_gprs.name(i) != grpContents[i]) {
424 dump();
425 CRASH();
426 }
427 }
428 for (FPRReg i = fpr0; i < numberOfFPRs; next(i)) {
429 if (m_fprs.isLocked(i) || m_fprs.name(i) != frpContents[i]) {
430 dump();
431 CRASH();
432 }
433 }
434 }
435 #endif
436
GPRTemporary(JITCodeGenerator * jit)437 GPRTemporary::GPRTemporary(JITCodeGenerator* jit)
438 : m_jit(jit)
439 , m_gpr(InvalidGPRReg)
440 {
441 m_gpr = m_jit->allocate();
442 }
443
GPRTemporary(JITCodeGenerator * jit,SpeculateIntegerOperand & op1)444 GPRTemporary::GPRTemporary(JITCodeGenerator* jit, SpeculateIntegerOperand& op1)
445 : m_jit(jit)
446 , m_gpr(InvalidGPRReg)
447 {
448 // locking into a register may free for reuse!
449 op1.gpr();
450 if (m_jit->canReuse(op1.index()))
451 m_gpr = m_jit->reuse(op1.gpr());
452 else
453 m_gpr = m_jit->allocate();
454 }
455
GPRTemporary(JITCodeGenerator * jit,SpeculateIntegerOperand & op1,SpeculateIntegerOperand & op2)456 GPRTemporary::GPRTemporary(JITCodeGenerator* jit, SpeculateIntegerOperand& op1, SpeculateIntegerOperand& op2)
457 : m_jit(jit)
458 , m_gpr(InvalidGPRReg)
459 {
460 // locking into a register may free for reuse!
461 op1.gpr();
462 op2.gpr();
463 if (m_jit->canReuse(op1.index()))
464 m_gpr = m_jit->reuse(op1.gpr());
465 else if (m_jit->canReuse(op2.index()))
466 m_gpr = m_jit->reuse(op2.gpr());
467 else
468 m_gpr = m_jit->allocate();
469 }
470
GPRTemporary(JITCodeGenerator * jit,IntegerOperand & op1)471 GPRTemporary::GPRTemporary(JITCodeGenerator* jit, IntegerOperand& op1)
472 : m_jit(jit)
473 , m_gpr(InvalidGPRReg)
474 {
475 // locking into a register may free for reuse!
476 op1.gpr();
477 if (m_jit->canReuse(op1.index()))
478 m_gpr = m_jit->reuse(op1.gpr());
479 else
480 m_gpr = m_jit->allocate();
481 }
482
GPRTemporary(JITCodeGenerator * jit,IntegerOperand & op1,IntegerOperand & op2)483 GPRTemporary::GPRTemporary(JITCodeGenerator* jit, IntegerOperand& op1, IntegerOperand& op2)
484 : m_jit(jit)
485 , m_gpr(InvalidGPRReg)
486 {
487 // locking into a register may free for reuse!
488 op1.gpr();
489 op2.gpr();
490 if (m_jit->canReuse(op1.index()))
491 m_gpr = m_jit->reuse(op1.gpr());
492 else if (m_jit->canReuse(op2.index()))
493 m_gpr = m_jit->reuse(op2.gpr());
494 else
495 m_gpr = m_jit->allocate();
496 }
497
GPRTemporary(JITCodeGenerator * jit,SpeculateCellOperand & op1)498 GPRTemporary::GPRTemporary(JITCodeGenerator* jit, SpeculateCellOperand& op1)
499 : m_jit(jit)
500 , m_gpr(InvalidGPRReg)
501 {
502 // locking into a register may free for reuse!
503 op1.gpr();
504 if (m_jit->canReuse(op1.index()))
505 m_gpr = m_jit->reuse(op1.gpr());
506 else
507 m_gpr = m_jit->allocate();
508 }
509
GPRTemporary(JITCodeGenerator * jit,JSValueOperand & op1)510 GPRTemporary::GPRTemporary(JITCodeGenerator* jit, JSValueOperand& op1)
511 : m_jit(jit)
512 , m_gpr(InvalidGPRReg)
513 {
514 // locking into a register may free for reuse!
515 op1.gpr();
516 if (m_jit->canReuse(op1.index()))
517 m_gpr = m_jit->reuse(op1.gpr());
518 else
519 m_gpr = m_jit->allocate();
520 }
521
FPRTemporary(JITCodeGenerator * jit)522 FPRTemporary::FPRTemporary(JITCodeGenerator* jit)
523 : m_jit(jit)
524 , m_fpr(InvalidFPRReg)
525 {
526 m_fpr = m_jit->fprAllocate();
527 }
528
FPRTemporary(JITCodeGenerator * jit,DoubleOperand & op1)529 FPRTemporary::FPRTemporary(JITCodeGenerator* jit, DoubleOperand& op1)
530 : m_jit(jit)
531 , m_fpr(InvalidFPRReg)
532 {
533 // locking into a register may free for reuse!
534 op1.fpr();
535 if (m_jit->canReuse(op1.index()))
536 m_fpr = m_jit->reuse(op1.fpr());
537 else
538 m_fpr = m_jit->fprAllocate();
539 }
540
FPRTemporary(JITCodeGenerator * jit,DoubleOperand & op1,DoubleOperand & op2)541 FPRTemporary::FPRTemporary(JITCodeGenerator* jit, DoubleOperand& op1, DoubleOperand& op2)
542 : m_jit(jit)
543 , m_fpr(InvalidFPRReg)
544 {
545 // locking into a register may free for reuse!
546 op1.fpr();
547 op2.fpr();
548 if (m_jit->canReuse(op1.index()))
549 m_fpr = m_jit->reuse(op1.fpr());
550 else if (m_jit->canReuse(op2.index()))
551 m_fpr = m_jit->reuse(op2.fpr());
552 else
553 m_fpr = m_jit->fprAllocate();
554 }
555
556 } } // namespace JSC::DFG
557
558 #endif
559