root/lj_asm_ppc.h

/* [<][>][^][v][top][bottom][index][help] */

INCLUDED FROM


DEFINITIONS

This source file includes following definitions.
  1. ra_hintalloc
  2. ra_alloc2
  3. asm_exitstub_setup
  4. asm_exitstub_addr
  5. asm_guardcc
  6. noconflict
  7. asm_fuseabase
  8. asm_fuseahuref
  9. asm_fusexref
  10. asm_fusexrefx
  11. asm_fusemadd
  12. asm_gencall
  13. asm_setupresult
  14. asm_call
  15. asm_callx
  16. asm_callid
  17. asm_retf
  18. asm_tointg
  19. asm_tobit
  20. asm_conv
  21. asm_conv64
  22. asm_strto
  23. asm_tvptr
  24. asm_tostr
  25. asm_aref
  26. asm_href
  27. asm_hrefk
  28. asm_newref
  29. asm_uref
  30. asm_fref
  31. asm_strref
  32. asm_fxloadins
  33. asm_fxstoreins
  34. asm_fload
  35. asm_fstore
  36. asm_xload
  37. asm_xstore
  38. asm_ahuvload
  39. asm_ahustore
  40. asm_sload
  41. asm_cnew
  42. asm_tbar
  43. asm_obar
  44. asm_fparith
  45. asm_fpunary
  46. asm_fpjoin_pow
  47. asm_add
  48. asm_sub
  49. asm_mul
  50. asm_neg
  51. asm_arithov
  52. asm_add64
  53. asm_sub64
  54. asm_neg64
  55. asm_bitnot
  56. asm_bitswap
  57. asm_bitop
  58. asm_fuseandsh
  59. asm_bitand
  60. asm_bitshift
  61. asm_min_max
  62. asm_intcomp_
  63. asm_comp
  64. asm_comp64
  65. asm_hiop
  66. asm_stack_check
  67. asm_stack_restore
  68. asm_gc_check
  69. asm_loop_fixup
  70. asm_head_root_base
  71. asm_head_side_base
  72. asm_tail_fixup
  73. asm_tail_prep
  74. asm_ir
  75. asm_setup_call_slots
  76. asm_setup_target
  77. lj_asm_patchexit

   1 /*
   2 ** PPC IR assembler (SSA IR -> machine code).
   3 ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5 
   6 /* -- Register allocator extensions --------------------------------------- */
   7 
   8 /* Allocate a register with a hint. */
   9 static Reg ra_hintalloc(ASMState *as, IRRef ref, Reg hint, RegSet allow)
  10 {
  11   Reg r = IR(ref)->r;
  12   if (ra_noreg(r)) {
  13     if (!ra_hashint(r) && !iscrossref(as, ref))
  14       ra_sethint(IR(ref)->r, hint);  /* Propagate register hint. */
  15     r = ra_allocref(as, ref, allow);
  16   }
  17   ra_noweak(as, r);
  18   return r;
  19 }
  20 
  21 /* Allocate two source registers for three-operand instructions. */
  22 static Reg ra_alloc2(ASMState *as, IRIns *ir, RegSet allow)
  23 {
  24   IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
  25   Reg left = irl->r, right = irr->r;
  26   if (ra_hasreg(left)) {
  27     ra_noweak(as, left);
  28     if (ra_noreg(right))
  29       right = ra_allocref(as, ir->op2, rset_exclude(allow, left));
  30     else
  31       ra_noweak(as, right);
  32   } else if (ra_hasreg(right)) {
  33     ra_noweak(as, right);
  34     left = ra_allocref(as, ir->op1, rset_exclude(allow, right));
  35   } else if (ra_hashint(right)) {
  36     right = ra_allocref(as, ir->op2, allow);
  37     left = ra_alloc1(as, ir->op1, rset_exclude(allow, right));
  38   } else {
  39     left = ra_allocref(as, ir->op1, allow);
  40     right = ra_alloc1(as, ir->op2, rset_exclude(allow, left));
  41   }
  42   return left | (right << 8);
  43 }
  44 
  45 /* -- Guard handling ------------------------------------------------------ */
  46 
  47 /* Setup exit stubs after the end of each trace. */
  48 static void asm_exitstub_setup(ASMState *as, ExitNo nexits)
  49 {
  50   ExitNo i;
  51   MCode *mxp = as->mctop;
  52   if (mxp - (nexits + 3 + MCLIM_REDZONE) < as->mclim)
  53     asm_mclimit(as);
  54   /* 1: mflr r0; bl ->vm_exit_handler; li r0, traceno; bl <1; bl <1; ... */
  55   for (i = nexits-1; (int32_t)i >= 0; i--)
  56     *--mxp = PPCI_BL|(((-3-i)&0x00ffffffu)<<2);
  57   *--mxp = PPCI_LI|PPCF_T(RID_TMP)|as->T->traceno;  /* Read by exit handler. */
  58   mxp--;
  59   *mxp = PPCI_BL|((((MCode *)(void *)lj_vm_exit_handler-mxp)&0x00ffffffu)<<2);
  60   *--mxp = PPCI_MFLR|PPCF_T(RID_TMP);
  61   as->mctop = mxp;
  62 }
  63 
  64 static MCode *asm_exitstub_addr(ASMState *as, ExitNo exitno)
  65 {
  66   /* Keep this in-sync with exitstub_trace_addr(). */
  67   return as->mctop + exitno + 3;
  68 }
  69 
  70 /* Emit conditional branch to exit for guard. */
  71 static void asm_guardcc(ASMState *as, PPCCC cc)
  72 {
  73   MCode *target = asm_exitstub_addr(as, as->snapno);
  74   MCode *p = as->mcp;
  75   if (LJ_UNLIKELY(p == as->invmcp)) {
  76     as->loopinv = 1;
  77     *p = PPCI_B | (((target-p) & 0x00ffffffu) << 2);
  78     emit_condbranch(as, PPCI_BC, cc^4, p);
  79     return;
  80   }
  81   emit_condbranch(as, PPCI_BC, cc, target);
  82 }
  83 
  84 /* -- Operand fusion ------------------------------------------------------ */
  85 
  86 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
  87 #define CONFLICT_SEARCH_LIM     31
  88 
  89 /* Check if there's no conflicting instruction between curins and ref. */
  90 static int noconflict(ASMState *as, IRRef ref, IROp conflict)
  91 {
  92   IRIns *ir = as->ir;
  93   IRRef i = as->curins;
  94   if (i > ref + CONFLICT_SEARCH_LIM)
  95     return 0;  /* Give up, ref is too far away. */
  96   while (--i > ref)
  97     if (ir[i].o == conflict)
  98       return 0;  /* Conflict found. */
  99   return 1;  /* Ok, no conflict. */
 100 }
 101 
 102 /* Fuse the array base of colocated arrays. */
 103 static int32_t asm_fuseabase(ASMState *as, IRRef ref)
 104 {
 105   IRIns *ir = IR(ref);
 106   if (ir->o == IR_TNEW && ir->op1 <= LJ_MAX_COLOSIZE &&
 107       !neverfuse(as) && noconflict(as, ref, IR_NEWREF))
 108     return (int32_t)sizeof(GCtab);
 109   return 0;
 110 }
 111 
 112 /* Indicates load/store indexed is ok. */
 113 #define AHUREF_LSX      ((int32_t)0x80000000)
 114 
 115 /* Fuse array/hash/upvalue reference into register+offset operand. */
 116 static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow)
 117 {
 118   IRIns *ir = IR(ref);
 119   if (ra_noreg(ir->r)) {
 120     if (ir->o == IR_AREF) {
 121       if (mayfuse(as, ref)) {
 122         if (irref_isk(ir->op2)) {
 123           IRRef tab = IR(ir->op1)->op1;
 124           int32_t ofs = asm_fuseabase(as, tab);
 125           IRRef refa = ofs ? tab : ir->op1;
 126           ofs += 8*IR(ir->op2)->i;
 127           if (checki16(ofs)) {
 128             *ofsp = ofs;
 129             return ra_alloc1(as, refa, allow);
 130           }
 131         }
 132         if (*ofsp == AHUREF_LSX) {
 133           Reg base = ra_alloc1(as, ir->op1, allow);
 134           Reg idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base));
 135           return base | (idx << 8);
 136         }
 137       }
 138     } else if (ir->o == IR_HREFK) {
 139       if (mayfuse(as, ref)) {
 140         int32_t ofs = (int32_t)(IR(ir->op2)->op2 * sizeof(Node));
 141         if (checki16(ofs)) {
 142           *ofsp = ofs;
 143           return ra_alloc1(as, ir->op1, allow);
 144         }
 145       }
 146     } else if (ir->o == IR_UREFC) {
 147       if (irref_isk(ir->op1)) {
 148         GCfunc *fn = ir_kfunc(IR(ir->op1));
 149         int32_t ofs = i32ptr(&gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.tv);
 150         int32_t jgl = (intptr_t)J2G(as->J);
 151         if ((uint32_t)(ofs-jgl) < 65536) {
 152           *ofsp = ofs-jgl-32768;
 153           return RID_JGL;
 154         } else {
 155           *ofsp = (int16_t)ofs;
 156           return ra_allock(as, ofs-(int16_t)ofs, allow);
 157         }
 158       }
 159     }
 160   }
 161   *ofsp = 0;
 162   return ra_alloc1(as, ref, allow);
 163 }
 164 
 165 /* Fuse XLOAD/XSTORE reference into load/store operand. */
 166 static void asm_fusexref(ASMState *as, PPCIns pi, Reg rt, IRRef ref,
 167                          RegSet allow, int32_t ofs)
 168 {
 169   IRIns *ir = IR(ref);
 170   Reg base;
 171   if (ra_noreg(ir->r) && canfuse(as, ir)) {
 172     if (ir->o == IR_ADD) {
 173       int32_t ofs2;
 174       if (irref_isk(ir->op2) && (ofs2 = ofs + IR(ir->op2)->i, checki16(ofs2))) {
 175         ofs = ofs2;
 176         ref = ir->op1;
 177       } else if (ofs == 0) {
 178         Reg right, left = ra_alloc2(as, ir, allow);
 179         right = (left >> 8); left &= 255;
 180         emit_fab(as, PPCI_LWZX | ((pi >> 20) & 0x780), rt, left, right);
 181         return;
 182       }
 183     } else if (ir->o == IR_STRREF) {
 184       lua_assert(ofs == 0);
 185       ofs = (int32_t)sizeof(GCstr);
 186       if (irref_isk(ir->op2)) {
 187         ofs += IR(ir->op2)->i;
 188         ref = ir->op1;
 189       } else if (irref_isk(ir->op1)) {
 190         ofs += IR(ir->op1)->i;
 191         ref = ir->op2;
 192       } else {
 193         /* NYI: Fuse ADD with constant. */
 194         Reg tmp, right, left = ra_alloc2(as, ir, allow);
 195         right = (left >> 8); left &= 255;
 196         tmp = ra_scratch(as, rset_exclude(rset_exclude(allow, left), right));
 197         emit_fai(as, pi, rt, tmp, ofs);
 198         emit_tab(as, PPCI_ADD, tmp, left, right);
 199         return;
 200       }
 201       if (!checki16(ofs)) {
 202         Reg left = ra_alloc1(as, ref, allow);
 203         Reg right = ra_allock(as, ofs, rset_exclude(allow, left));
 204         emit_fab(as, PPCI_LWZX | ((pi >> 20) & 0x780), rt, left, right);
 205         return;
 206       }
 207     }
 208   }
 209   base = ra_alloc1(as, ref, allow);
 210   emit_fai(as, pi, rt, base, ofs);
 211 }
 212 
 213 /* Fuse XLOAD/XSTORE reference into indexed-only load/store operand. */
 214 static void asm_fusexrefx(ASMState *as, PPCIns pi, Reg rt, IRRef ref,
 215                           RegSet allow)
 216 {
 217   IRIns *ira = IR(ref);
 218   Reg right, left;
 219   if (canfuse(as, ira) && ira->o == IR_ADD && ra_noreg(ira->r)) {
 220     left = ra_alloc2(as, ira, allow);
 221     right = (left >> 8); left &= 255;
 222   } else {
 223     right = ra_alloc1(as, ref, allow);
 224     left = RID_R0;
 225   }
 226   emit_tab(as, pi, rt, left, right);
 227 }
 228 
 229 /* Fuse to multiply-add/sub instruction. */
 230 static int asm_fusemadd(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pir)
 231 {
 232   IRRef lref = ir->op1, rref = ir->op2;
 233   IRIns *irm;
 234   if (lref != rref &&
 235       ((mayfuse(as, lref) && (irm = IR(lref), irm->o == IR_MUL) &&
 236         ra_noreg(irm->r)) ||
 237        (mayfuse(as, rref) && (irm = IR(rref), irm->o == IR_MUL) &&
 238         (rref = lref, pi = pir, ra_noreg(irm->r))))) {
 239     Reg dest = ra_dest(as, ir, RSET_FPR);
 240     Reg add = ra_alloc1(as, rref, RSET_FPR);
 241     Reg right, left = ra_alloc2(as, irm, rset_exclude(RSET_FPR, add));
 242     right = (left >> 8); left &= 255;
 243     emit_facb(as, pi, dest, left, right, add);
 244     return 1;
 245   }
 246   return 0;
 247 }
 248 
 249 /* -- Calls --------------------------------------------------------------- */
 250 
 251 /* Generate a call to a C function. */
 252 static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
 253 {
 254   uint32_t n, nargs = CCI_NARGS(ci);
 255   int32_t ofs = 8;
 256   Reg gpr = REGARG_FIRSTGPR, fpr = REGARG_FIRSTFPR;
 257   if ((void *)ci->func)
 258     emit_call(as, (void *)ci->func);
 259   for (n = 0; n < nargs; n++) {  /* Setup args. */
 260     IRRef ref = args[n];
 261     if (ref) {
 262       IRIns *ir = IR(ref);
 263       if (irt_isfp(ir->t)) {
 264         if (fpr <= REGARG_LASTFPR) {
 265           lua_assert(rset_test(as->freeset, fpr));  /* Already evicted. */
 266           ra_leftov(as, fpr, ref);
 267           fpr++;
 268         } else {
 269           Reg r = ra_alloc1(as, ref, RSET_FPR);
 270           if (irt_isnum(ir->t)) ofs = (ofs + 4) & ~4;
 271           emit_spstore(as, ir, r, ofs);
 272           ofs += irt_isnum(ir->t) ? 8 : 4;
 273         }
 274       } else {
 275         if (gpr <= REGARG_LASTGPR) {
 276           lua_assert(rset_test(as->freeset, gpr));  /* Already evicted. */
 277           ra_leftov(as, gpr, ref);
 278           gpr++;
 279         } else {
 280           Reg r = ra_alloc1(as, ref, RSET_GPR);
 281           emit_spstore(as, ir, r, ofs);
 282           ofs += 4;
 283         }
 284       }
 285     } else {
 286       if (gpr <= REGARG_LASTGPR)
 287         gpr++;
 288       else
 289         ofs += 4;
 290     }
 291     checkmclim(as);
 292   }
 293   if ((ci->flags & CCI_VARARG))  /* Vararg calls need to know about FPR use. */
 294     emit_tab(as, fpr == REGARG_FIRSTFPR ? PPCI_CRXOR : PPCI_CREQV, 6, 6, 6);
 295 }
 296 
 297 /* Setup result reg/sp for call. Evict scratch regs. */
 298 static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
 299 {
 300   RegSet drop = RSET_SCRATCH;
 301   int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t));
 302   if ((ci->flags & CCI_NOFPRCLOBBER))
 303     drop &= ~RSET_FPR;
 304   if (ra_hasreg(ir->r))
 305     rset_clear(drop, ir->r);  /* Dest reg handled below. */
 306   if (hiop && ra_hasreg((ir+1)->r))
 307     rset_clear(drop, (ir+1)->r);  /* Dest reg handled below. */
 308   ra_evictset(as, drop);  /* Evictions must be performed first. */
 309   if (ra_used(ir)) {
 310     lua_assert(!irt_ispri(ir->t));
 311     if (irt_isfp(ir->t)) {
 312       if ((ci->flags & CCI_CASTU64)) {
 313         /* Use spill slot or temp slots. */
 314         int32_t ofs = ir->s ? sps_scale(ir->s) : SPOFS_TMP;
 315         Reg dest = ir->r;
 316         if (ra_hasreg(dest)) {
 317           ra_free(as, dest);
 318           ra_modified(as, dest);
 319           emit_fai(as, PPCI_LFD, dest, RID_SP, ofs);
 320         }
 321         emit_tai(as, PPCI_STW, RID_RETHI, RID_SP, ofs);
 322         emit_tai(as, PPCI_STW, RID_RETLO, RID_SP, ofs+4);
 323       } else {
 324         ra_destreg(as, ir, RID_FPRET);
 325       }
 326     } else if (hiop) {
 327       ra_destpair(as, ir);
 328     } else {
 329       ra_destreg(as, ir, RID_RET);
 330     }
 331   }
 332 }
 333 
 334 static void asm_call(ASMState *as, IRIns *ir)
 335 {
 336   IRRef args[CCI_NARGS_MAX];
 337   const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
 338   asm_collectargs(as, ir, ci, args);
 339   asm_setupresult(as, ir, ci);
 340   asm_gencall(as, ci, args);
 341 }
 342 
 343 static void asm_callx(ASMState *as, IRIns *ir)
 344 {
 345   IRRef args[CCI_NARGS_MAX*2];
 346   CCallInfo ci;
 347   IRRef func;
 348   IRIns *irf;
 349   ci.flags = asm_callx_flags(as, ir);
 350   asm_collectargs(as, ir, &ci, args);
 351   asm_setupresult(as, ir, &ci);
 352   func = ir->op2; irf = IR(func);
 353   if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); }
 354   if (irref_isk(func)) {  /* Call to constant address. */
 355     ci.func = (ASMFunction)(void *)(irf->i);
 356   } else {  /* Need a non-argument register for indirect calls. */
 357     RegSet allow = RSET_GPR & ~RSET_RANGE(RID_R0, REGARG_LASTGPR+1);
 358     Reg freg = ra_alloc1(as, func, allow);
 359     *--as->mcp = PPCI_BCTRL;
 360     *--as->mcp = PPCI_MTCTR | PPCF_T(freg);
 361     ci.func = (ASMFunction)(void *)0;
 362   }
 363   asm_gencall(as, &ci, args);
 364 }
 365 
 366 static void asm_callid(ASMState *as, IRIns *ir, IRCallID id)
 367 {
 368   const CCallInfo *ci = &lj_ir_callinfo[id];
 369   IRRef args[2];
 370   args[0] = ir->op1;
 371   args[1] = ir->op2;
 372   asm_setupresult(as, ir, ci);
 373   asm_gencall(as, ci, args);
 374 }
 375 
 376 /* -- Returns ------------------------------------------------------------- */
 377 
 378 /* Return to lower frame. Guard that it goes to the right spot. */
 379 static void asm_retf(ASMState *as, IRIns *ir)
 380 {
 381   Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
 382   void *pc = ir_kptr(IR(ir->op2));
 383   int32_t delta = 1+bc_a(*((const BCIns *)pc - 1));
 384   as->topslot -= (BCReg)delta;
 385   if ((int32_t)as->topslot < 0) as->topslot = 0;
 386   irt_setmark(IR(REF_BASE)->t);  /* Children must not coalesce with BASE reg. */
 387   emit_setgl(as, base, jit_base);
 388   emit_addptr(as, base, -8*delta);
 389   asm_guardcc(as, CC_NE);
 390   emit_ab(as, PPCI_CMPW, RID_TMP,
 391           ra_allock(as, i32ptr(pc), rset_exclude(RSET_GPR, base)));
 392   emit_tai(as, PPCI_LWZ, RID_TMP, base, -8);
 393 }
 394 
 395 /* -- Type conversions ---------------------------------------------------- */
 396 
 397 static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
 398 {
 399   RegSet allow = RSET_FPR;
 400   Reg tmp = ra_scratch(as, rset_clear(allow, left));
 401   Reg fbias = ra_scratch(as, rset_clear(allow, tmp));
 402   Reg dest = ra_dest(as, ir, RSET_GPR);
 403   Reg hibias = ra_allock(as, 0x43300000, rset_exclude(RSET_GPR, dest));
 404   asm_guardcc(as, CC_NE);
 405   emit_fab(as, PPCI_FCMPU, 0, tmp, left);
 406   emit_fab(as, PPCI_FSUB, tmp, tmp, fbias);
 407   emit_fai(as, PPCI_LFD, tmp, RID_SP, SPOFS_TMP);
 408   emit_tai(as, PPCI_STW, RID_TMP, RID_SP, SPOFS_TMPLO);
 409   emit_tai(as, PPCI_STW, hibias, RID_SP, SPOFS_TMPHI);
 410   emit_asi(as, PPCI_XORIS, RID_TMP, dest, 0x8000);
 411   emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
 412   emit_lsptr(as, PPCI_LFS, (fbias & 31),
 413              (void *)lj_ir_k64_find(as->J, U64x(59800004,59800000)),
 414              RSET_GPR);
 415   emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
 416   emit_fb(as, PPCI_FCTIWZ, tmp, left);
 417 }
 418 
 419 static void asm_tobit(ASMState *as, IRIns *ir)
 420 {
 421   RegSet allow = RSET_FPR;
 422   Reg dest = ra_dest(as, ir, RSET_GPR);
 423   Reg left = ra_alloc1(as, ir->op1, allow);
 424   Reg right = ra_alloc1(as, ir->op2, rset_clear(allow, left));
 425   Reg tmp = ra_scratch(as, rset_clear(allow, right));
 426   emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
 427   emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
 428   emit_fab(as, PPCI_FADD, tmp, left, right);
 429 }
 430 
 431 static void asm_conv(ASMState *as, IRIns *ir)
 432 {
 433   IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
 434   int stfp = (st == IRT_NUM || st == IRT_FLOAT);
 435   IRRef lref = ir->op1;
 436   lua_assert(irt_type(ir->t) != st);
 437   lua_assert(!(irt_isint64(ir->t) ||
 438                (st == IRT_I64 || st == IRT_U64))); /* Handled by SPLIT. */
 439   if (irt_isfp(ir->t)) {
 440     Reg dest = ra_dest(as, ir, RSET_FPR);
 441     if (stfp) {  /* FP to FP conversion. */
 442       if (st == IRT_NUM)  /* double -> float conversion. */
 443         emit_fb(as, PPCI_FRSP, dest, ra_alloc1(as, lref, RSET_FPR));
 444       else  /* float -> double conversion is a no-op on PPC. */
 445         ra_leftov(as, dest, lref);  /* Do nothing, but may need to move regs. */
 446     } else {  /* Integer to FP conversion. */
 447       /* IRT_INT: Flip hibit, bias with 2^52, subtract 2^52+2^31. */
 448       /* IRT_U32: Bias with 2^52, subtract 2^52. */
 449       RegSet allow = RSET_GPR;
 450       Reg left = ra_alloc1(as, lref, allow);
 451       Reg hibias = ra_allock(as, 0x43300000, rset_clear(allow, left));
 452       Reg fbias = ra_scratch(as, rset_exclude(RSET_FPR, dest));
 453       const float *kbias;
 454       if (irt_isfloat(ir->t)) emit_fb(as, PPCI_FRSP, dest, dest);
 455       emit_fab(as, PPCI_FSUB, dest, dest, fbias);
 456       emit_fai(as, PPCI_LFD, dest, RID_SP, SPOFS_TMP);
 457       kbias = (const float *)lj_ir_k64_find(as->J, U64x(59800004,59800000));
 458       if (st == IRT_U32) kbias++;
 459       emit_lsptr(as, PPCI_LFS, (fbias & 31), (void *)kbias,
 460                  rset_clear(allow, hibias));
 461       emit_tai(as, PPCI_STW, st == IRT_U32 ? left : RID_TMP,
 462                RID_SP, SPOFS_TMPLO);
 463       emit_tai(as, PPCI_STW, hibias, RID_SP, SPOFS_TMPHI);
 464       if (st != IRT_U32) emit_asi(as, PPCI_XORIS, RID_TMP, left, 0x8000);
 465     }
 466   } else if (stfp) {  /* FP to integer conversion. */
 467     if (irt_isguard(ir->t)) {
 468       /* Checked conversions are only supported from number to int. */
 469       lua_assert(irt_isint(ir->t) && st == IRT_NUM);
 470       asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
 471     } else {
 472       Reg dest = ra_dest(as, ir, RSET_GPR);
 473       Reg left = ra_alloc1(as, lref, RSET_FPR);
 474       Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
 475       if (irt_isu32(ir->t)) {
 476         /* Convert both x and x-2^31 to int and merge results. */
 477         Reg tmpi = ra_scratch(as, rset_exclude(RSET_GPR, dest));
 478         emit_asb(as, PPCI_OR, dest, dest, tmpi);  /* Select with mask idiom. */
 479         emit_asb(as, PPCI_AND, tmpi, tmpi, RID_TMP);
 480         emit_asb(as, PPCI_ANDC, dest, dest, RID_TMP);
 481         emit_tai(as, PPCI_LWZ, tmpi, RID_SP, SPOFS_TMPLO);  /* tmp = (int)(x) */
 482         emit_tai(as, PPCI_ADDIS, dest, dest, 0x8000);  /* dest += 2^31 */
 483         emit_asb(as, PPCI_SRAWI, RID_TMP, dest, 31);  /* mask = -(dest < 0) */
 484         emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
 485         emit_tai(as, PPCI_LWZ, dest,
 486                  RID_SP, SPOFS_TMPLO);  /* dest = (int)(x-2^31) */
 487         emit_fb(as, PPCI_FCTIWZ, tmp, left);
 488         emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
 489         emit_fb(as, PPCI_FCTIWZ, tmp, tmp);
 490         emit_fab(as, PPCI_FSUB, tmp, left, tmp);
 491         emit_lsptr(as, PPCI_LFS, (tmp & 31),
 492                    (void *)lj_ir_k64_find(as->J, U64x(4f000000,00000000)),
 493                    RSET_GPR);
 494       } else {
 495         emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
 496         emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
 497         emit_fb(as, PPCI_FCTIWZ, tmp, left);
 498       }
 499     }
 500   } else {
 501     Reg dest = ra_dest(as, ir, RSET_GPR);
 502     if (st >= IRT_I8 && st <= IRT_U16) {  /* Extend to 32 bit integer. */
 503       Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
 504       lua_assert(irt_isint(ir->t) || irt_isu32(ir->t));
 505       if ((ir->op2 & IRCONV_SEXT))
 506         emit_as(as, st == IRT_I8 ? PPCI_EXTSB : PPCI_EXTSH, dest, left);
 507       else
 508         emit_rot(as, PPCI_RLWINM, dest, left, 0, st == IRT_U8 ? 24 : 16, 31);
 509     } else {  /* 32/64 bit integer conversions. */
 510       /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
 511       ra_leftov(as, dest, lref);  /* Do nothing, but may need to move regs. */
 512     }
 513   }
 514 }
 515 
 516 #if LJ_HASFFI
 517 static void asm_conv64(ASMState *as, IRIns *ir)
 518 {
 519   IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
 520   IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
 521   IRCallID id;
 522   const CCallInfo *ci;
 523   IRRef args[2];
 524   args[0] = ir->op1;
 525   args[1] = (ir-1)->op1;
 526   if (st == IRT_NUM || st == IRT_FLOAT) {
 527     id = IRCALL_fp64_d2l + ((st == IRT_FLOAT) ? 2 : 0) + (dt - IRT_I64);
 528     ir--;
 529   } else {
 530     id = IRCALL_fp64_l2d + ((dt == IRT_FLOAT) ? 2 : 0) + (st - IRT_I64);
 531   }
 532   ci = &lj_ir_callinfo[id];
 533   asm_setupresult(as, ir, ci);
 534   asm_gencall(as, ci, args);
 535 }
 536 #endif
 537 
 538 static void asm_strto(ASMState *as, IRIns *ir)
 539 {
 540   const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
 541   IRRef args[2];
 542   int32_t ofs;
 543   RegSet drop = RSET_SCRATCH;
 544   if (ra_hasreg(ir->r)) rset_set(drop, ir->r);  /* Spill dest reg (if any). */
 545   ra_evictset(as, drop);
 546   asm_guardcc(as, CC_EQ);
 547   emit_ai(as, PPCI_CMPWI, RID_RET, 0);  /* Test return status. */
 548   args[0] = ir->op1;      /* GCstr *str */
 549   args[1] = ASMREF_TMP1;  /* TValue *n  */
 550   asm_gencall(as, ci, args);
 551   /* Store the result to the spill slot or temp slots. */
 552   ofs = ir->s ? sps_scale(ir->s) : SPOFS_TMP;
 553   emit_tai(as, PPCI_ADDI, ra_releasetmp(as, ASMREF_TMP1), RID_SP, ofs);
 554 }
 555 
 556 /* Get pointer to TValue. */
 557 static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
 558 {
 559   IRIns *ir = IR(ref);
 560   if (irt_isnum(ir->t)) {
 561     if (irref_isk(ref))  /* Use the number constant itself as a TValue. */
 562       ra_allockreg(as, i32ptr(ir_knum(ir)), dest);
 563     else  /* Otherwise force a spill and use the spill slot. */
 564       emit_tai(as, PPCI_ADDI, dest, RID_SP, ra_spill(as, ir));
 565   } else {
 566     /* Otherwise use g->tmptv to hold the TValue. */
 567     RegSet allow = rset_exclude(RSET_GPR, dest);
 568     Reg type;
 569     emit_tai(as, PPCI_ADDI, dest, RID_JGL, offsetof(global_State, tmptv)-32768);
 570     if (!irt_ispri(ir->t)) {
 571       Reg src = ra_alloc1(as, ref, allow);
 572       emit_setgl(as, src, tmptv.gcr);
 573     }
 574     type = ra_allock(as, irt_toitype(ir->t), allow);
 575     emit_setgl(as, type, tmptv.it);
 576   }
 577 }
 578 
 579 static void asm_tostr(ASMState *as, IRIns *ir)
 580 {
 581   IRRef args[2];
 582   args[0] = ASMREF_L;
 583   as->gcsteps++;
 584   if (irt_isnum(IR(ir->op1)->t) || (ir+1)->o == IR_HIOP) {
 585     const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromnum];
 586     args[1] = ASMREF_TMP1;  /* const lua_Number * */
 587     asm_setupresult(as, ir, ci);  /* GCstr * */
 588     asm_gencall(as, ci, args);
 589     asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op1);
 590   } else {
 591     const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromint];
 592     args[1] = ir->op1;  /* int32_t k */
 593     asm_setupresult(as, ir, ci);  /* GCstr * */
 594     asm_gencall(as, ci, args);
 595   }
 596 }
 597 
 598 /* -- Memory references --------------------------------------------------- */
 599 
 600 static void asm_aref(ASMState *as, IRIns *ir)
 601 {
 602   Reg dest = ra_dest(as, ir, RSET_GPR);
 603   Reg idx, base;
 604   if (irref_isk(ir->op2)) {
 605     IRRef tab = IR(ir->op1)->op1;
 606     int32_t ofs = asm_fuseabase(as, tab);
 607     IRRef refa = ofs ? tab : ir->op1;
 608     ofs += 8*IR(ir->op2)->i;
 609     if (checki16(ofs)) {
 610       base = ra_alloc1(as, refa, RSET_GPR);
 611       emit_tai(as, PPCI_ADDI, dest, base, ofs);
 612       return;
 613     }
 614   }
 615   base = ra_alloc1(as, ir->op1, RSET_GPR);
 616   idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base));
 617   emit_tab(as, PPCI_ADD, dest, RID_TMP, base);
 618   emit_slwi(as, RID_TMP, idx, 3);
 619 }
 620 
 621 /* Inlined hash lookup. Specialized for key type and for const keys.
 622 ** The equivalent C code is:
 623 **   Node *n = hashkey(t, key);
 624 **   do {
 625 **     if (lj_obj_equal(&n->key, key)) return &n->val;
 626 **   } while ((n = nextnode(n)));
 627 **   return niltv(L);
 628 */
 629 static void asm_href(ASMState *as, IRIns *ir, IROp merge)
 630 {
 631   RegSet allow = RSET_GPR;
 632   int destused = ra_used(ir);
 633   Reg dest = ra_dest(as, ir, allow);
 634   Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest));
 635   Reg key = RID_NONE, tmp1 = RID_TMP, tmp2;
 636   Reg tisnum = RID_NONE, tmpnum = RID_NONE;
 637   IRRef refkey = ir->op2;
 638   IRIns *irkey = IR(refkey);
 639   IRType1 kt = irkey->t;
 640   uint32_t khash;
 641   MCLabel l_end, l_loop, l_next;
 642 
 643   rset_clear(allow, tab);
 644   if (irt_isnum(kt)) {
 645     key = ra_alloc1(as, refkey, RSET_FPR);
 646     tmpnum = ra_scratch(as, rset_exclude(RSET_FPR, key));
 647     tisnum = ra_allock(as, (int32_t)LJ_TISNUM, allow);
 648     rset_clear(allow, tisnum);
 649   } else if (!irt_ispri(kt)) {
 650     key = ra_alloc1(as, refkey, allow);
 651     rset_clear(allow, key);
 652   }
 653   tmp2 = ra_scratch(as, allow);
 654   rset_clear(allow, tmp2);
 655 
 656   /* Key not found in chain: jump to exit (if merged) or load niltv. */
 657   l_end = emit_label(as);
 658   as->invmcp = NULL;
 659   if (merge == IR_NE)
 660     asm_guardcc(as, CC_EQ);
 661   else if (destused)
 662     emit_loada(as, dest, niltvg(J2G(as->J)));
 663 
 664   /* Follow hash chain until the end. */
 665   l_loop = --as->mcp;
 666   emit_ai(as, PPCI_CMPWI, dest, 0);
 667   emit_tai(as, PPCI_LWZ, dest, dest, (int32_t)offsetof(Node, next));
 668   l_next = emit_label(as);
 669 
 670   /* Type and value comparison. */
 671   if (merge == IR_EQ)
 672     asm_guardcc(as, CC_EQ);
 673   else
 674     emit_condbranch(as, PPCI_BC|PPCF_Y, CC_EQ, l_end);
 675   if (irt_isnum(kt)) {
 676     emit_fab(as, PPCI_FCMPU, 0, tmpnum, key);
 677     emit_condbranch(as, PPCI_BC, CC_GE, l_next);
 678     emit_ab(as, PPCI_CMPLW, tmp1, tisnum);
 679     emit_fai(as, PPCI_LFD, tmpnum, dest, (int32_t)offsetof(Node, key.n));
 680   } else {
 681     if (!irt_ispri(kt)) {
 682       emit_ab(as, PPCI_CMPW, tmp2, key);
 683       emit_condbranch(as, PPCI_BC, CC_NE, l_next);
 684     }
 685     emit_ai(as, PPCI_CMPWI, tmp1, irt_toitype(irkey->t));
 686     if (!irt_ispri(kt))
 687       emit_tai(as, PPCI_LWZ, tmp2, dest, (int32_t)offsetof(Node, key.gcr));
 688   }
 689   emit_tai(as, PPCI_LWZ, tmp1, dest, (int32_t)offsetof(Node, key.it));
 690   *l_loop = PPCI_BC | PPCF_Y | PPCF_CC(CC_NE) |
 691             (((char *)as->mcp-(char *)l_loop) & 0xffffu);
 692 
 693   /* Load main position relative to tab->node into dest. */
 694   khash = irref_isk(refkey) ? ir_khash(irkey) : 1;
 695   if (khash == 0) {
 696     emit_tai(as, PPCI_LWZ, dest, tab, (int32_t)offsetof(GCtab, node));
 697   } else {
 698     Reg tmphash = tmp1;
 699     if (irref_isk(refkey))
 700       tmphash = ra_allock(as, khash, allow);
 701     emit_tab(as, PPCI_ADD, dest, dest, tmp1);
 702     emit_tai(as, PPCI_MULLI, tmp1, tmp1, sizeof(Node));
 703     emit_asb(as, PPCI_AND, tmp1, tmp2, tmphash);
 704     emit_tai(as, PPCI_LWZ, dest, tab, (int32_t)offsetof(GCtab, node));
 705     emit_tai(as, PPCI_LWZ, tmp2, tab, (int32_t)offsetof(GCtab, hmask));
 706     if (irref_isk(refkey)) {
 707       /* Nothing to do. */
 708     } else if (irt_isstr(kt)) {
 709       emit_tai(as, PPCI_LWZ, tmp1, key, (int32_t)offsetof(GCstr, hash));
 710     } else {  /* Must match with hash*() in lj_tab.c. */
 711       emit_tab(as, PPCI_SUBF, tmp1, tmp2, tmp1);
 712       emit_rotlwi(as, tmp2, tmp2, HASH_ROT3);
 713       emit_asb(as, PPCI_XOR, tmp1, tmp1, tmp2);
 714       emit_rotlwi(as, tmp1, tmp1, (HASH_ROT2+HASH_ROT1)&31);
 715       emit_tab(as, PPCI_SUBF, tmp2, dest, tmp2);
 716       if (irt_isnum(kt)) {
 717         int32_t ofs = ra_spill(as, irkey);
 718         emit_asb(as, PPCI_XOR, tmp2, tmp2, tmp1);
 719         emit_rotlwi(as, dest, tmp1, HASH_ROT1);
 720         emit_tab(as, PPCI_ADD, tmp1, tmp1, tmp1);
 721         emit_tai(as, PPCI_LWZ, tmp2, RID_SP, ofs+4);
 722         emit_tai(as, PPCI_LWZ, tmp1, RID_SP, ofs);
 723       } else {
 724         emit_asb(as, PPCI_XOR, tmp2, key, tmp1);
 725         emit_rotlwi(as, dest, tmp1, HASH_ROT1);
 726         emit_tai(as, PPCI_ADDI, tmp1, tmp2, HASH_BIAS);
 727         emit_tai(as, PPCI_ADDIS, tmp2, key, (HASH_BIAS + 32768)>>16);
 728       }
 729     }
 730   }
 731 }
 732 
 733 static void asm_hrefk(ASMState *as, IRIns *ir)
 734 {
 735   IRIns *kslot = IR(ir->op2);
 736   IRIns *irkey = IR(kslot->op1);
 737   int32_t ofs = (int32_t)(kslot->op2 * sizeof(Node));
 738   int32_t kofs = ofs + (int32_t)offsetof(Node, key);
 739   Reg dest = (ra_used(ir)||ofs > 32736) ? ra_dest(as, ir, RSET_GPR) : RID_NONE;
 740   Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
 741   Reg key = RID_NONE, type = RID_TMP, idx = node;
 742   RegSet allow = rset_exclude(RSET_GPR, node);
 743   lua_assert(ofs % sizeof(Node) == 0);
 744   if (ofs > 32736) {
 745     idx = dest;
 746     rset_clear(allow, dest);
 747     kofs = (int32_t)offsetof(Node, key);
 748   } else if (ra_hasreg(dest)) {
 749     emit_tai(as, PPCI_ADDI, dest, node, ofs);
 750   }
 751   asm_guardcc(as, CC_NE);
 752   if (!irt_ispri(irkey->t)) {
 753     key = ra_scratch(as, allow);
 754     rset_clear(allow, key);
 755   }
 756   rset_clear(allow, type);
 757   if (irt_isnum(irkey->t)) {
 758     emit_cmpi(as, key, (int32_t)ir_knum(irkey)->u32.lo);
 759     asm_guardcc(as, CC_NE);
 760     emit_cmpi(as, type, (int32_t)ir_knum(irkey)->u32.hi);
 761   } else {
 762     if (ra_hasreg(key)) {
 763       emit_cmpi(as, key, irkey->i);  /* May use RID_TMP, i.e. type. */
 764       asm_guardcc(as, CC_NE);
 765     }
 766     emit_ai(as, PPCI_CMPWI, type, irt_toitype(irkey->t));
 767   }
 768   if (ra_hasreg(key)) emit_tai(as, PPCI_LWZ, key, idx, kofs+4);
 769   emit_tai(as, PPCI_LWZ, type, idx, kofs);
 770   if (ofs > 32736) {
 771     emit_tai(as, PPCI_ADDIS, dest, dest, (ofs + 32768) >> 16);
 772     emit_tai(as, PPCI_ADDI, dest, node, ofs);
 773   }
 774 }
 775 
 776 static void asm_newref(ASMState *as, IRIns *ir)
 777 {
 778   const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey];
 779   IRRef args[3];
 780   if (ir->r == RID_SINK)
 781     return;
 782   args[0] = ASMREF_L;     /* lua_State *L */
 783   args[1] = ir->op1;      /* GCtab *t     */
 784   args[2] = ASMREF_TMP1;  /* cTValue *key */
 785   asm_setupresult(as, ir, ci);  /* TValue * */
 786   asm_gencall(as, ci, args);
 787   asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2);
 788 }
 789 
 790 static void asm_uref(ASMState *as, IRIns *ir)
 791 {
 792   Reg dest = ra_dest(as, ir, RSET_GPR);
 793   if (irref_isk(ir->op1)) {
 794     GCfunc *fn = ir_kfunc(IR(ir->op1));
 795     MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
 796     emit_lsptr(as, PPCI_LWZ, dest, v, RSET_GPR);
 797   } else {
 798     Reg uv = ra_scratch(as, RSET_GPR);
 799     Reg func = ra_alloc1(as, ir->op1, RSET_GPR);
 800     if (ir->o == IR_UREFC) {
 801       asm_guardcc(as, CC_NE);
 802       emit_ai(as, PPCI_CMPWI, RID_TMP, 1);
 803       emit_tai(as, PPCI_ADDI, dest, uv, (int32_t)offsetof(GCupval, tv));
 804       emit_tai(as, PPCI_LBZ, RID_TMP, uv, (int32_t)offsetof(GCupval, closed));
 805     } else {
 806       emit_tai(as, PPCI_LWZ, dest, uv, (int32_t)offsetof(GCupval, v));
 807     }
 808     emit_tai(as, PPCI_LWZ, uv, func,
 809              (int32_t)offsetof(GCfuncL, uvptr) + 4*(int32_t)(ir->op2 >> 8));
 810   }
 811 }
 812 
 813 static void asm_fref(ASMState *as, IRIns *ir)
 814 {
 815   UNUSED(as); UNUSED(ir);
 816   lua_assert(!ra_used(ir));
 817 }
 818 
 819 static void asm_strref(ASMState *as, IRIns *ir)
 820 {
 821   Reg dest = ra_dest(as, ir, RSET_GPR);
 822   IRRef ref = ir->op2, refk = ir->op1;
 823   int32_t ofs = (int32_t)sizeof(GCstr);
 824   Reg r;
 825   if (irref_isk(ref)) {
 826     IRRef tmp = refk; refk = ref; ref = tmp;
 827   } else if (!irref_isk(refk)) {
 828     Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
 829     IRIns *irr = IR(ir->op2);
 830     if (ra_hasreg(irr->r)) {
 831       ra_noweak(as, irr->r);
 832       right = irr->r;
 833     } else if (mayfuse(as, irr->op2) &&
 834                irr->o == IR_ADD && irref_isk(irr->op2) &&
 835                checki16(ofs + IR(irr->op2)->i)) {
 836       ofs += IR(irr->op2)->i;
 837       right = ra_alloc1(as, irr->op1, rset_exclude(RSET_GPR, left));
 838     } else {
 839       right = ra_allocref(as, ir->op2, rset_exclude(RSET_GPR, left));
 840     }
 841     emit_tai(as, PPCI_ADDI, dest, dest, ofs);
 842     emit_tab(as, PPCI_ADD, dest, left, right);
 843     return;
 844   }
 845   r = ra_alloc1(as, ref, RSET_GPR);
 846   ofs += IR(refk)->i;
 847   if (checki16(ofs))
 848     emit_tai(as, PPCI_ADDI, dest, r, ofs);
 849   else
 850     emit_tab(as, PPCI_ADD, dest, r,
 851              ra_allock(as, ofs, rset_exclude(RSET_GPR, r)));
 852 }
 853 
 854 /* -- Loads and stores ---------------------------------------------------- */
 855 
 856 static PPCIns asm_fxloadins(IRIns *ir)
 857 {
 858   switch (irt_type(ir->t)) {
 859   case IRT_I8: return PPCI_LBZ;  /* Needs sign-extension. */
 860   case IRT_U8: return PPCI_LBZ;
 861   case IRT_I16: return PPCI_LHA;
 862   case IRT_U16: return PPCI_LHZ;
 863   case IRT_NUM: return PPCI_LFD;
 864   case IRT_FLOAT: return PPCI_LFS;
 865   default: return PPCI_LWZ;
 866   }
 867 }
 868 
 869 static PPCIns asm_fxstoreins(IRIns *ir)
 870 {
 871   switch (irt_type(ir->t)) {
 872   case IRT_I8: case IRT_U8: return PPCI_STB;
 873   case IRT_I16: case IRT_U16: return PPCI_STH;
 874   case IRT_NUM: return PPCI_STFD;
 875   case IRT_FLOAT: return PPCI_STFS;
 876   default: return PPCI_STW;
 877   }
 878 }
 879 
 880 static void asm_fload(ASMState *as, IRIns *ir)
 881 {
 882   Reg dest = ra_dest(as, ir, RSET_GPR);
 883   Reg idx = ra_alloc1(as, ir->op1, RSET_GPR);
 884   PPCIns pi = asm_fxloadins(ir);
 885   int32_t ofs;
 886   if (ir->op2 == IRFL_TAB_ARRAY) {
 887     ofs = asm_fuseabase(as, ir->op1);
 888     if (ofs) {  /* Turn the t->array load into an add for colocated arrays. */
 889       emit_tai(as, PPCI_ADDI, dest, idx, ofs);
 890       return;
 891     }
 892   }
 893   ofs = field_ofs[ir->op2];
 894   lua_assert(!irt_isi8(ir->t));
 895   emit_tai(as, pi, dest, idx, ofs);
 896 }
 897 
 898 static void asm_fstore(ASMState *as, IRIns *ir)
 899 {
 900   if (ir->r != RID_SINK) {
 901     Reg src = ra_alloc1(as, ir->op2, RSET_GPR);
 902     IRIns *irf = IR(ir->op1);
 903     Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src));
 904     int32_t ofs = field_ofs[irf->op2];
 905     PPCIns pi = asm_fxstoreins(ir);
 906     emit_tai(as, pi, src, idx, ofs);
 907   }
 908 }
 909 
 910 static void asm_xload(ASMState *as, IRIns *ir)
 911 {
 912   Reg dest = ra_dest(as, ir, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
 913   lua_assert(!(ir->op2 & IRXLOAD_UNALIGNED));
 914   if (irt_isi8(ir->t))
 915     emit_as(as, PPCI_EXTSB, dest, dest);
 916   asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR, 0);
 917 }
 918 
 919 static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs)
 920 {
 921   IRIns *irb;
 922   if (ir->r == RID_SINK)
 923     return;
 924   if (ofs == 0 && mayfuse(as, ir->op2) && (irb = IR(ir->op2))->o == IR_BSWAP &&
 925       ra_noreg(irb->r) && (irt_isint(ir->t) || irt_isu32(ir->t))) {
 926     /* Fuse BSWAP with XSTORE to stwbrx. */
 927     Reg src = ra_alloc1(as, irb->op1, RSET_GPR);
 928     asm_fusexrefx(as, PPCI_STWBRX, src, ir->op1, rset_exclude(RSET_GPR, src));
 929   } else {
 930     Reg src = ra_alloc1(as, ir->op2, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
 931     asm_fusexref(as, asm_fxstoreins(ir), src, ir->op1,
 932                  rset_exclude(RSET_GPR, src), ofs);
 933   }
 934 }
 935 
 936 static void asm_ahuvload(ASMState *as, IRIns *ir)
 937 {
 938   IRType1 t = ir->t;
 939   Reg dest = RID_NONE, type = RID_TMP, tmp = RID_TMP, idx;
 940   RegSet allow = RSET_GPR;
 941   int32_t ofs = AHUREF_LSX;
 942   if (ra_used(ir)) {
 943     lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t));
 944     if (!irt_isnum(t)) ofs = 0;
 945     dest = ra_dest(as, ir, irt_isnum(t) ? RSET_FPR : RSET_GPR);
 946     rset_clear(allow, dest);
 947   }
 948   idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
 949   if (irt_isnum(t)) {
 950     Reg tisnum = ra_allock(as, (int32_t)LJ_TISNUM, rset_exclude(allow, idx));
 951     asm_guardcc(as, CC_GE);
 952     emit_ab(as, PPCI_CMPLW, type, tisnum);
 953     if (ra_hasreg(dest)) {
 954       if (ofs == AHUREF_LSX) {
 955         tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR,
 956                                                        (idx&255)), (idx>>8)));
 957         emit_fab(as, PPCI_LFDX, dest, (idx&255), tmp);
 958       } else {
 959         emit_fai(as, PPCI_LFD, dest, idx, ofs);
 960       }
 961     }
 962   } else {
 963     asm_guardcc(as, CC_NE);
 964     emit_ai(as, PPCI_CMPWI, type, irt_toitype(t));
 965     if (ra_hasreg(dest)) emit_tai(as, PPCI_LWZ, dest, idx, ofs+4);
 966   }
 967   if (ofs == AHUREF_LSX) {
 968     emit_tab(as, PPCI_LWZX, type, (idx&255), tmp);
 969     emit_slwi(as, tmp, (idx>>8), 3);
 970   } else {
 971     emit_tai(as, PPCI_LWZ, type, idx, ofs);
 972   }
 973 }
 974 
 975 static void asm_ahustore(ASMState *as, IRIns *ir)
 976 {
 977   RegSet allow = RSET_GPR;
 978   Reg idx, src = RID_NONE, type = RID_NONE;
 979   int32_t ofs = AHUREF_LSX;
 980   if (ir->r == RID_SINK)
 981     return;
 982   if (irt_isnum(ir->t)) {
 983     src = ra_alloc1(as, ir->op2, RSET_FPR);
 984   } else {
 985     if (!irt_ispri(ir->t)) {
 986       src = ra_alloc1(as, ir->op2, allow);
 987       rset_clear(allow, src);
 988       ofs = 0;
 989     }
 990     type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
 991     rset_clear(allow, type);
 992   }
 993   idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
 994   if (irt_isnum(ir->t)) {
 995     if (ofs == AHUREF_LSX) {
 996       emit_fab(as, PPCI_STFDX, src, (idx&255), RID_TMP);
 997       emit_slwi(as, RID_TMP, (idx>>8), 3);
 998     } else {
 999       emit_fai(as, PPCI_STFD, src, idx, ofs);
1000     }
1001   } else {
1002     if (ra_hasreg(src))
1003       emit_tai(as, PPCI_STW, src, idx, ofs+4);
1004     if (ofs == AHUREF_LSX) {
1005       emit_tab(as, PPCI_STWX, type, (idx&255), RID_TMP);
1006       emit_slwi(as, RID_TMP, (idx>>8), 3);
1007     } else {
1008       emit_tai(as, PPCI_STW, type, idx, ofs);
1009     }
1010   }
1011 }
1012 
1013 static void asm_sload(ASMState *as, IRIns *ir)
1014 {
1015   int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 0 : 4);
1016   IRType1 t = ir->t;
1017   Reg dest = RID_NONE, type = RID_NONE, base;
1018   RegSet allow = RSET_GPR;
1019   lua_assert(!(ir->op2 & IRSLOAD_PARENT));  /* Handled by asm_head_side(). */
1020   lua_assert(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK));
1021   lua_assert(LJ_DUALNUM ||
1022              !irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME)));
1023   if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
1024     dest = ra_scratch(as, RSET_FPR);
1025     asm_tointg(as, ir, dest);
1026     t.irt = IRT_NUM;  /* Continue with a regular number type check. */
1027   } else if (ra_used(ir)) {
1028     lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t));
1029     dest = ra_dest(as, ir, irt_isnum(t) ? RSET_FPR : RSET_GPR);
1030     rset_clear(allow, dest);
1031     base = ra_alloc1(as, REF_BASE, allow);
1032     rset_clear(allow, base);
1033     if ((ir->op2 & IRSLOAD_CONVERT)) {
1034       if (irt_isint(t)) {
1035         emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
1036         dest = ra_scratch(as, RSET_FPR);
1037         emit_fai(as, PPCI_STFD, dest, RID_SP, SPOFS_TMP);
1038         emit_fb(as, PPCI_FCTIWZ, dest, dest);
1039         t.irt = IRT_NUM;  /* Check for original type. */
1040       } else {
1041         Reg tmp = ra_scratch(as, allow);
1042         Reg hibias = ra_allock(as, 0x43300000, rset_clear(allow, tmp));
1043         Reg fbias = ra_scratch(as, rset_exclude(RSET_FPR, dest));
1044         emit_fab(as, PPCI_FSUB, dest, dest, fbias);
1045         emit_fai(as, PPCI_LFD, dest, RID_SP, SPOFS_TMP);
1046         emit_lsptr(as, PPCI_LFS, (fbias & 31),
1047                    (void *)lj_ir_k64_find(as->J, U64x(59800004,59800000)),
1048                    rset_clear(allow, hibias));
1049         emit_tai(as, PPCI_STW, tmp, RID_SP, SPOFS_TMPLO);
1050         emit_tai(as, PPCI_STW, hibias, RID_SP, SPOFS_TMPHI);
1051         emit_asi(as, PPCI_XORIS, tmp, tmp, 0x8000);
1052         dest = tmp;
1053         t.irt = IRT_INT;  /* Check for original type. */
1054       }
1055     }
1056     goto dotypecheck;
1057   }
1058   base = ra_alloc1(as, REF_BASE, allow);
1059   rset_clear(allow, base);
1060 dotypecheck:
1061   if (irt_isnum(t)) {
1062     if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1063       Reg tisnum = ra_allock(as, (int32_t)LJ_TISNUM, allow);
1064       asm_guardcc(as, CC_GE);
1065       emit_ab(as, PPCI_CMPLW, RID_TMP, tisnum);
1066       type = RID_TMP;
1067     }
1068     if (ra_hasreg(dest)) emit_fai(as, PPCI_LFD, dest, base, ofs-4);
1069   } else {
1070     if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1071       asm_guardcc(as, CC_NE);
1072       emit_ai(as, PPCI_CMPWI, RID_TMP, irt_toitype(t));
1073       type = RID_TMP;
1074     }
1075     if (ra_hasreg(dest)) emit_tai(as, PPCI_LWZ, dest, base, ofs);
1076   }
1077   if (ra_hasreg(type)) emit_tai(as, PPCI_LWZ, type, base, ofs-4);
1078 }
1079 
1080 /* -- Allocations --------------------------------------------------------- */
1081 
1082 #if LJ_HASFFI
1083 static void asm_cnew(ASMState *as, IRIns *ir)
1084 {
1085   CTState *cts = ctype_ctsG(J2G(as->J));
1086   CTypeID ctypeid = (CTypeID)IR(ir->op1)->i;
1087   CTSize sz = (ir->o == IR_CNEWI || ir->op2 == REF_NIL) ?
1088               lj_ctype_size(cts, ctypeid) : (CTSize)IR(ir->op2)->i;
1089   const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1090   IRRef args[2];
1091   RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1092   RegSet drop = RSET_SCRATCH;
1093   lua_assert(sz != CTSIZE_INVALID);
1094 
1095   args[0] = ASMREF_L;     /* lua_State *L */
1096   args[1] = ASMREF_TMP1;  /* MSize size   */
1097   as->gcsteps++;
1098 
1099   if (ra_hasreg(ir->r))
1100     rset_clear(drop, ir->r);  /* Dest reg handled below. */
1101   ra_evictset(as, drop);
1102   if (ra_used(ir))
1103     ra_destreg(as, ir, RID_RET);  /* GCcdata * */
1104 
1105   /* Initialize immutable cdata object. */
1106   if (ir->o == IR_CNEWI) {
1107     int32_t ofs = sizeof(GCcdata);
1108     lua_assert(sz == 4 || sz == 8);
1109     if (sz == 8) {
1110       ofs += 4;
1111       lua_assert((ir+1)->o == IR_HIOP);
1112     }
1113     for (;;) {
1114       Reg r = ra_alloc1(as, ir->op2, allow);
1115       emit_tai(as, PPCI_STW, r, RID_RET, ofs);
1116       rset_clear(allow, r);
1117       if (ofs == sizeof(GCcdata)) break;
1118       ofs -= 4; ir++;
1119     }
1120   }
1121   /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1122   emit_tai(as, PPCI_STB, RID_RET+1, RID_RET, offsetof(GCcdata, gct));
1123   emit_tai(as, PPCI_STH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid));
1124   emit_ti(as, PPCI_LI, RID_RET+1, ~LJ_TCDATA);
1125   emit_ti(as, PPCI_LI, RID_TMP, ctypeid);  /* Lower 16 bit used. Sign-ext ok. */
1126   asm_gencall(as, ci, args);
1127   ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
1128                ra_releasetmp(as, ASMREF_TMP1));
1129 }
1130 #else
1131 #define asm_cnew(as, ir)        ((void)0)
1132 #endif
1133 
1134 /* -- Write barriers ------------------------------------------------------ */
1135 
1136 static void asm_tbar(ASMState *as, IRIns *ir)
1137 {
1138   Reg tab = ra_alloc1(as, ir->op1, RSET_GPR);
1139   Reg mark = ra_scratch(as, rset_exclude(RSET_GPR, tab));
1140   Reg link = RID_TMP;
1141   MCLabel l_end = emit_label(as);
1142   emit_tai(as, PPCI_STW, link, tab, (int32_t)offsetof(GCtab, gclist));
1143   emit_tai(as, PPCI_STB, mark, tab, (int32_t)offsetof(GCtab, marked));
1144   emit_setgl(as, tab, gc.grayagain);
1145   lua_assert(LJ_GC_BLACK == 0x04);
1146   emit_rot(as, PPCI_RLWINM, mark, mark, 0, 30, 28);  /* Clear black bit. */
1147   emit_getgl(as, link, gc.grayagain);
1148   emit_condbranch(as, PPCI_BC|PPCF_Y, CC_EQ, l_end);
1149   emit_asi(as, PPCI_ANDIDOT, RID_TMP, mark, LJ_GC_BLACK);
1150   emit_tai(as, PPCI_LBZ, mark, tab, (int32_t)offsetof(GCtab, marked));
1151 }
1152 
1153 static void asm_obar(ASMState *as, IRIns *ir)
1154 {
1155   const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_barrieruv];
1156   IRRef args[2];
1157   MCLabel l_end;
1158   Reg obj, val, tmp;
1159   /* No need for other object barriers (yet). */
1160   lua_assert(IR(ir->op1)->o == IR_UREFC);
1161   ra_evictset(as, RSET_SCRATCH);
1162   l_end = emit_label(as);
1163   args[0] = ASMREF_TMP1;  /* global_State *g */
1164   args[1] = ir->op1;      /* TValue *tv      */
1165   asm_gencall(as, ci, args);
1166   emit_tai(as, PPCI_ADDI, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
1167   obj = IR(ir->op1)->r;
1168   tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj));
1169   emit_condbranch(as, PPCI_BC|PPCF_Y, CC_EQ, l_end);
1170   emit_asi(as, PPCI_ANDIDOT, tmp, tmp, LJ_GC_BLACK);
1171   emit_condbranch(as, PPCI_BC, CC_EQ, l_end);
1172   emit_asi(as, PPCI_ANDIDOT, RID_TMP, RID_TMP, LJ_GC_WHITES);
1173   val = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, obj));
1174   emit_tai(as, PPCI_LBZ, tmp, obj,
1175            (int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv));
1176   emit_tai(as, PPCI_LBZ, RID_TMP, val, (int32_t)offsetof(GChead, marked));
1177 }
1178 
1179 /* -- Arithmetic and logic operations ------------------------------------- */
1180 
1181 static void asm_fparith(ASMState *as, IRIns *ir, PPCIns pi)
1182 {
1183   Reg dest = ra_dest(as, ir, RSET_FPR);
1184   Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1185   right = (left >> 8); left &= 255;
1186   if (pi == PPCI_FMUL)
1187     emit_fac(as, pi, dest, left, right);
1188   else
1189     emit_fab(as, pi, dest, left, right);
1190 }
1191 
1192 static void asm_fpunary(ASMState *as, IRIns *ir, PPCIns pi)
1193 {
1194   Reg dest = ra_dest(as, ir, RSET_FPR);
1195   Reg left = ra_hintalloc(as, ir->op1, dest, RSET_FPR);
1196   emit_fb(as, pi, dest, left);
1197 }
1198 
1199 static int asm_fpjoin_pow(ASMState *as, IRIns *ir)
1200 {
1201   IRIns *irp = IR(ir->op1);
1202   if (irp == ir-1 && irp->o == IR_MUL && !ra_used(irp)) {
1203     IRIns *irpp = IR(irp->op1);
1204     if (irpp == ir-2 && irpp->o == IR_FPMATH &&
1205         irpp->op2 == IRFPM_LOG2 && !ra_used(irpp)) {
1206       const CCallInfo *ci = &lj_ir_callinfo[IRCALL_pow];
1207       IRRef args[2];
1208       args[0] = irpp->op1;
1209       args[1] = irp->op2;
1210       asm_setupresult(as, ir, ci);
1211       asm_gencall(as, ci, args);
1212       return 1;
1213     }
1214   }
1215   return 0;
1216 }
1217 
1218 static void asm_add(ASMState *as, IRIns *ir)
1219 {
1220   if (irt_isnum(ir->t)) {
1221     if (!asm_fusemadd(as, ir, PPCI_FMADD, PPCI_FMADD))
1222       asm_fparith(as, ir, PPCI_FADD);
1223   } else {
1224     Reg dest = ra_dest(as, ir, RSET_GPR);
1225     Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1226     PPCIns pi;
1227     if (irref_isk(ir->op2)) {
1228       int32_t k = IR(ir->op2)->i;
1229       if (checki16(k)) {
1230         pi = PPCI_ADDI;
1231         /* May fail due to spills/restores above, but simplifies the logic. */
1232         if (as->flagmcp == as->mcp) {
1233           as->flagmcp = NULL;
1234           as->mcp++;
1235           pi = PPCI_ADDICDOT;
1236         }
1237         emit_tai(as, pi, dest, left, k);
1238         return;
1239       } else if ((k & 0xffff) == 0) {
1240         emit_tai(as, PPCI_ADDIS, dest, left, (k >> 16));
1241         return;
1242       } else if (!as->sectref) {
1243         emit_tai(as, PPCI_ADDIS, dest, dest, (k + 32768) >> 16);
1244         emit_tai(as, PPCI_ADDI, dest, left, k);
1245         return;
1246       }
1247     }
1248     pi = PPCI_ADD;
1249     /* May fail due to spills/restores above, but simplifies the logic. */
1250     if (as->flagmcp == as->mcp) {
1251       as->flagmcp = NULL;
1252       as->mcp++;
1253       pi |= PPCF_DOT;
1254     }
1255     right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1256     emit_tab(as, pi, dest, left, right);
1257   }
1258 }
1259 
1260 static void asm_sub(ASMState *as, IRIns *ir)
1261 {
1262   if (irt_isnum(ir->t)) {
1263     if (!asm_fusemadd(as, ir, PPCI_FMSUB, PPCI_FNMSUB))
1264       asm_fparith(as, ir, PPCI_FSUB);
1265   } else {
1266     PPCIns pi = PPCI_SUBF;
1267     Reg dest = ra_dest(as, ir, RSET_GPR);
1268     Reg left, right;
1269     if (irref_isk(ir->op1)) {
1270       int32_t k = IR(ir->op1)->i;
1271       if (checki16(k)) {
1272         right = ra_alloc1(as, ir->op2, RSET_GPR);
1273         emit_tai(as, PPCI_SUBFIC, dest, right, k);
1274         return;
1275       }
1276     }
1277     /* May fail due to spills/restores above, but simplifies the logic. */
1278     if (as->flagmcp == as->mcp) {
1279       as->flagmcp = NULL;
1280       as->mcp++;
1281       pi |= PPCF_DOT;
1282     }
1283     left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1284     right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1285     emit_tab(as, pi, dest, right, left);  /* Subtract right _from_ left. */
1286   }
1287 }
1288 
1289 static void asm_mul(ASMState *as, IRIns *ir)
1290 {
1291   if (irt_isnum(ir->t)) {
1292     asm_fparith(as, ir, PPCI_FMUL);
1293   } else {
1294     PPCIns pi = PPCI_MULLW;
1295     Reg dest = ra_dest(as, ir, RSET_GPR);
1296     Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1297     if (irref_isk(ir->op2)) {
1298       int32_t k = IR(ir->op2)->i;
1299       if (checki16(k)) {
1300         emit_tai(as, PPCI_MULLI, dest, left, k);
1301         return;
1302       }
1303     }
1304     /* May fail due to spills/restores above, but simplifies the logic. */
1305     if (as->flagmcp == as->mcp) {
1306       as->flagmcp = NULL;
1307       as->mcp++;
1308       pi |= PPCF_DOT;
1309     }
1310     right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1311     emit_tab(as, pi, dest, left, right);
1312   }
1313 }
1314 
1315 static void asm_neg(ASMState *as, IRIns *ir)
1316 {
1317   if (irt_isnum(ir->t)) {
1318     asm_fpunary(as, ir, PPCI_FNEG);
1319   } else {
1320     Reg dest, left;
1321     PPCIns pi = PPCI_NEG;
1322     if (as->flagmcp == as->mcp) {
1323       as->flagmcp = NULL;
1324       as->mcp++;
1325       pi |= PPCF_DOT;
1326     }
1327     dest = ra_dest(as, ir, RSET_GPR);
1328     left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1329     emit_tab(as, pi, dest, left, 0);
1330   }
1331 }
1332 
1333 static void asm_arithov(ASMState *as, IRIns *ir, PPCIns pi)
1334 {
1335   Reg dest, left, right;
1336   if (as->flagmcp == as->mcp) {
1337     as->flagmcp = NULL;
1338     as->mcp++;
1339   }
1340   asm_guardcc(as, CC_SO);
1341   dest = ra_dest(as, ir, RSET_GPR);
1342   left = ra_alloc2(as, ir, RSET_GPR);
1343   right = (left >> 8); left &= 255;
1344   if (pi == PPCI_SUBFO) { Reg tmp = left; left = right; right = tmp; }
1345   emit_tab(as, pi|PPCF_DOT, dest, left, right);
1346 }
1347 
1348 #if LJ_HASFFI
1349 static void asm_add64(ASMState *as, IRIns *ir)
1350 {
1351   Reg dest = ra_dest(as, ir, RSET_GPR);
1352   Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
1353   PPCIns pi = PPCI_ADDE;
1354   if (irref_isk(ir->op2)) {
1355     int32_t k = IR(ir->op2)->i;
1356     if (k == 0)
1357       pi = PPCI_ADDZE;
1358     else if (k == -1)
1359       pi = PPCI_ADDME;
1360     else
1361       goto needright;
1362     right = 0;
1363   } else {
1364   needright:
1365     right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1366   }
1367   emit_tab(as, pi, dest, left, right);
1368   ir--;
1369   dest = ra_dest(as, ir, RSET_GPR);
1370   left = ra_alloc1(as, ir->op1, RSET_GPR);
1371   if (irref_isk(ir->op2)) {
1372     int32_t k = IR(ir->op2)->i;
1373     if (checki16(k)) {
1374       emit_tai(as, PPCI_ADDIC, dest, left, k);
1375       return;
1376     }
1377   }
1378   right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1379   emit_tab(as, PPCI_ADDC, dest, left, right);
1380 }
1381 
1382 static void asm_sub64(ASMState *as, IRIns *ir)
1383 {
1384   Reg dest = ra_dest(as, ir, RSET_GPR);
1385   Reg left, right = ra_alloc1(as, ir->op2, RSET_GPR);
1386   PPCIns pi = PPCI_SUBFE;
1387   if (irref_isk(ir->op1)) {
1388     int32_t k = IR(ir->op1)->i;
1389     if (k == 0)
1390       pi = PPCI_SUBFZE;
1391     else if (k == -1)
1392       pi = PPCI_SUBFME;
1393     else
1394       goto needleft;
1395     left = 0;
1396   } else {
1397   needleft:
1398     left = ra_alloc1(as, ir->op1, rset_exclude(RSET_GPR, right));
1399   }
1400   emit_tab(as, pi, dest, right, left);  /* Subtract right _from_ left. */
1401   ir--;
1402   dest = ra_dest(as, ir, RSET_GPR);
1403   right = ra_alloc1(as, ir->op2, RSET_GPR);
1404   if (irref_isk(ir->op1)) {
1405     int32_t k = IR(ir->op1)->i;
1406     if (checki16(k)) {
1407       emit_tai(as, PPCI_SUBFIC, dest, right, k);
1408       return;
1409     }
1410   }
1411   left = ra_alloc1(as, ir->op1, rset_exclude(RSET_GPR, right));
1412   emit_tab(as, PPCI_SUBFC, dest, right, left);
1413 }
1414 
1415 static void asm_neg64(ASMState *as, IRIns *ir)
1416 {
1417   Reg dest = ra_dest(as, ir, RSET_GPR);
1418   Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1419   emit_tab(as, PPCI_SUBFZE, dest, left, 0);
1420   ir--;
1421   dest = ra_dest(as, ir, RSET_GPR);
1422   left = ra_alloc1(as, ir->op1, RSET_GPR);
1423   emit_tai(as, PPCI_SUBFIC, dest, left, 0);
1424 }
1425 #endif
1426 
1427 static void asm_bitnot(ASMState *as, IRIns *ir)
1428 {
1429   Reg dest, left, right;
1430   PPCIns pi = PPCI_NOR;
1431   if (as->flagmcp == as->mcp) {
1432     as->flagmcp = NULL;
1433     as->mcp++;
1434     pi |= PPCF_DOT;
1435   }
1436   dest = ra_dest(as, ir, RSET_GPR);
1437   if (mayfuse(as, ir->op1)) {
1438     IRIns *irl = IR(ir->op1);
1439     if (irl->o == IR_BAND)
1440       pi ^= (PPCI_NOR ^ PPCI_NAND);
1441     else if (irl->o == IR_BXOR)
1442       pi ^= (PPCI_NOR ^ PPCI_EQV);
1443     else if (irl->o != IR_BOR)
1444       goto nofuse;
1445     left = ra_hintalloc(as, irl->op1, dest, RSET_GPR);
1446     right = ra_alloc1(as, irl->op2, rset_exclude(RSET_GPR, left));
1447   } else {
1448 nofuse:
1449     left = right = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1450   }
1451   emit_asb(as, pi, dest, left, right);
1452 }
1453 
1454 static void asm_bitswap(ASMState *as, IRIns *ir)
1455 {
1456   Reg dest = ra_dest(as, ir, RSET_GPR);
1457   IRIns *irx;
1458   if (mayfuse(as, ir->op1) && (irx = IR(ir->op1))->o == IR_XLOAD &&
1459       ra_noreg(irx->r) && (irt_isint(irx->t) || irt_isu32(irx->t))) {
1460     /* Fuse BSWAP with XLOAD to lwbrx. */
1461     asm_fusexrefx(as, PPCI_LWBRX, dest, irx->op1, RSET_GPR);
1462   } else {
1463     Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1464     Reg tmp = dest;
1465     if (tmp == left) {
1466       tmp = RID_TMP;
1467       emit_mr(as, dest, RID_TMP);
1468     }
1469     emit_rot(as, PPCI_RLWIMI, tmp, left, 24, 16, 23);
1470     emit_rot(as, PPCI_RLWIMI, tmp, left, 24, 0, 7);
1471     emit_rotlwi(as, tmp, left, 8);
1472   }
1473 }
1474 
1475 static void asm_bitop(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik)
1476 {
1477   Reg dest = ra_dest(as, ir, RSET_GPR);
1478   Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1479   if (irref_isk(ir->op2)) {
1480     int32_t k = IR(ir->op2)->i;
1481     Reg tmp = left;
1482     if ((checku16(k) || (k & 0xffff) == 0) || (tmp = dest, !as->sectref)) {
1483       if (!checku16(k)) {
1484         emit_asi(as, pik ^ (PPCI_ORI ^ PPCI_ORIS), dest, tmp, (k >> 16));
1485         if ((k & 0xffff) == 0) return;
1486       }
1487       emit_asi(as, pik, dest, left, k);
1488       return;
1489     }
1490   }
1491   /* May fail due to spills/restores above, but simplifies the logic. */
1492   if (as->flagmcp == as->mcp) {
1493     as->flagmcp = NULL;
1494     as->mcp++;
1495     pi |= PPCF_DOT;
1496   }
1497   right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1498   emit_asb(as, pi, dest, left, right);
1499 }
1500 
1501 /* Fuse BAND with contiguous bitmask and a shift to rlwinm. */
1502 static void asm_fuseandsh(ASMState *as, PPCIns pi, int32_t mask, IRRef ref)
1503 {
1504   IRIns *ir;
1505   Reg left;
1506   if (mayfuse(as, ref) && (ir = IR(ref), ra_noreg(ir->r)) &&
1507       irref_isk(ir->op2) && ir->o >= IR_BSHL && ir->o <= IR_BROR) {
1508     int32_t sh = (IR(ir->op2)->i & 31);
1509     switch (ir->o) {
1510     case IR_BSHL:
1511       if ((mask & ((1u<<sh)-1))) goto nofuse;
1512       break;
1513     case IR_BSHR:
1514       if ((mask & ~((~0u)>>sh))) goto nofuse;
1515       sh = ((32-sh)&31);
1516       break;
1517     case IR_BROL:
1518       break;
1519     default:
1520       goto nofuse;
1521     }
1522     left = ra_alloc1(as, ir->op1, RSET_GPR);
1523     *--as->mcp = pi | PPCF_T(left) | PPCF_B(sh);
1524     return;
1525   }
1526 nofuse:
1527   left = ra_alloc1(as, ref, RSET_GPR);
1528   *--as->mcp = pi | PPCF_T(left);
1529 }
1530 
1531 static void asm_bitand(ASMState *as, IRIns *ir)
1532 {
1533   Reg dest, left, right;
1534   IRRef lref = ir->op1;
1535   PPCIns dot = 0;
1536   IRRef op2;
1537   if (as->flagmcp == as->mcp) {
1538     as->flagmcp = NULL;
1539     as->mcp++;
1540     dot = PPCF_DOT;
1541   }
1542   dest = ra_dest(as, ir, RSET_GPR);
1543   if (irref_isk(ir->op2)) {
1544     int32_t k = IR(ir->op2)->i;
1545     if (k) {
1546       /* First check for a contiguous bitmask as used by rlwinm. */
1547       uint32_t s1 = lj_ffs((uint32_t)k);
1548       uint32_t k1 = ((uint32_t)k >> s1);
1549       if ((k1 & (k1+1)) == 0) {
1550         asm_fuseandsh(as, PPCI_RLWINM|dot | PPCF_A(dest) |
1551                           PPCF_MB(31-lj_fls((uint32_t)k)) | PPCF_ME(31-s1),
1552                           k, lref);
1553         return;
1554       }
1555       if (~(uint32_t)k) {
1556         uint32_t s2 = lj_ffs(~(uint32_t)k);
1557         uint32_t k2 = (~(uint32_t)k >> s2);
1558         if ((k2 & (k2+1)) == 0) {
1559           asm_fuseandsh(as, PPCI_RLWINM|dot | PPCF_A(dest) |
1560                             PPCF_MB(32-s2) | PPCF_ME(30-lj_fls(~(uint32_t)k)),
1561                             k, lref);
1562           return;
1563         }
1564       }
1565     }
1566     if (checku16(k)) {
1567       left = ra_alloc1(as, lref, RSET_GPR);
1568       emit_asi(as, PPCI_ANDIDOT, dest, left, k);
1569       return;
1570     } else if ((k & 0xffff) == 0) {
1571       left = ra_alloc1(as, lref, RSET_GPR);
1572       emit_asi(as, PPCI_ANDISDOT, dest, left, (k >> 16));
1573       return;
1574     }
1575   }
1576   op2 = ir->op2;
1577   if (mayfuse(as, op2) && IR(op2)->o == IR_BNOT && ra_noreg(IR(op2)->r)) {
1578     dot ^= (PPCI_AND ^ PPCI_ANDC);
1579     op2 = IR(op2)->op1;
1580   }
1581   left = ra_hintalloc(as, lref, dest, RSET_GPR);
1582   right = ra_alloc1(as, op2, rset_exclude(RSET_GPR, left));
1583   emit_asb(as, PPCI_AND ^ dot, dest, left, right);
1584 }
1585 
1586 static void asm_bitshift(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik)
1587 {
1588   Reg dest, left;
1589   Reg dot = 0;
1590   if (as->flagmcp == as->mcp) {
1591     as->flagmcp = NULL;
1592     as->mcp++;
1593     dot = PPCF_DOT;
1594   }
1595   dest = ra_dest(as, ir, RSET_GPR);
1596   left = ra_alloc1(as, ir->op1, RSET_GPR);
1597   if (irref_isk(ir->op2)) {  /* Constant shifts. */
1598     int32_t shift = (IR(ir->op2)->i & 31);
1599     if (pik == 0)  /* SLWI */
1600       emit_rot(as, PPCI_RLWINM|dot, dest, left, shift, 0, 31-shift);
1601     else if (pik == 1)  /* SRWI */
1602       emit_rot(as, PPCI_RLWINM|dot, dest, left, (32-shift)&31, shift, 31);
1603     else
1604       emit_asb(as, pik|dot, dest, left, shift);
1605   } else {
1606     Reg right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1607     emit_asb(as, pi|dot, dest, left, right);
1608   }
1609 }
1610 
1611 static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
1612 {
1613   if (irt_isnum(ir->t)) {
1614     Reg dest = ra_dest(as, ir, RSET_FPR);
1615     Reg tmp = dest;
1616     Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1617     right = (left >> 8); left &= 255;
1618     if (tmp == left || tmp == right)
1619       tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_FPR,
1620                                         dest), left), right));
1621     emit_facb(as, PPCI_FSEL, dest, tmp,
1622               ismax ? left : right, ismax ? right : left);
1623     emit_fab(as, PPCI_FSUB, tmp, left, right);
1624   } else {
1625     Reg dest = ra_dest(as, ir, RSET_GPR);
1626     Reg tmp1 = RID_TMP, tmp2 = dest;
1627     Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1628     right = (left >> 8); left &= 255;
1629     if (tmp2 == left || tmp2 == right)
1630       tmp2 = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_GPR,
1631                                          dest), left), right));
1632     emit_tab(as, PPCI_ADD, dest, tmp2, right);
1633     emit_asb(as, ismax ? PPCI_ANDC : PPCI_AND, tmp2, tmp2, tmp1);
1634     emit_tab(as, PPCI_SUBFE, tmp1, tmp1, tmp1);
1635     emit_tab(as, PPCI_SUBFC, tmp2, tmp2, tmp1);
1636     emit_asi(as, PPCI_XORIS, tmp2, right, 0x8000);
1637     emit_asi(as, PPCI_XORIS, tmp1, left, 0x8000);
1638   }
1639 }
1640 
1641 /* -- Comparisons --------------------------------------------------------- */
1642 
1643 #define CC_UNSIGNED     0x08    /* Unsigned integer comparison. */
1644 #define CC_TWO          0x80    /* Check two flags for FP comparison. */
1645 
1646 /* Map of comparisons to flags. ORDER IR. */
1647 static const uint8_t asm_compmap[IR_ABC+1] = {
1648   /* op     int cc                 FP cc */
1649   /* LT  */ CC_GE               + (CC_GE<<4),
1650   /* GE  */ CC_LT               + (CC_LE<<4) + CC_TWO,
1651   /* LE  */ CC_GT               + (CC_GE<<4) + CC_TWO,
1652   /* GT  */ CC_LE               + (CC_LE<<4),
1653   /* ULT */ CC_GE + CC_UNSIGNED + (CC_GT<<4) + CC_TWO,
1654   /* UGE */ CC_LT + CC_UNSIGNED + (CC_LT<<4),
1655   /* ULE */ CC_GT + CC_UNSIGNED + (CC_GT<<4),
1656   /* UGT */ CC_LE + CC_UNSIGNED + (CC_LT<<4) + CC_TWO,
1657   /* EQ  */ CC_NE               + (CC_NE<<4),
1658   /* NE  */ CC_EQ               + (CC_EQ<<4),
1659   /* ABC */ CC_LE + CC_UNSIGNED + (CC_LT<<4) + CC_TWO  /* Same as UGT. */
1660 };
1661 
1662 static void asm_intcomp_(ASMState *as, IRRef lref, IRRef rref, Reg cr, PPCCC cc)
1663 {
1664   Reg right, left = ra_alloc1(as, lref, RSET_GPR);
1665   if (irref_isk(rref)) {
1666     int32_t k = IR(rref)->i;
1667     if ((cc & CC_UNSIGNED) == 0) {  /* Signed comparison with constant. */
1668       if (checki16(k)) {
1669         emit_tai(as, PPCI_CMPWI, cr, left, k);
1670         /* Signed comparison with zero and referencing previous ins? */
1671         if (k == 0 && lref == as->curins-1)
1672           as->flagmcp = as->mcp;  /* Allow elimination of the compare. */
1673         return;
1674       } else if ((cc & 3) == (CC_EQ & 3)) {  /* Use CMPLWI for EQ or NE. */
1675         if (checku16(k)) {
1676           emit_tai(as, PPCI_CMPLWI, cr, left, k);
1677           return;
1678         } else if (!as->sectref && ra_noreg(IR(rref)->r)) {
1679           emit_tai(as, PPCI_CMPLWI, cr, RID_TMP, k);
1680           emit_asi(as, PPCI_XORIS, RID_TMP, left, (k >> 16));
1681           return;
1682         }
1683       }
1684     } else {  /* Unsigned comparison with constant. */
1685       if (checku16(k)) {
1686         emit_tai(as, PPCI_CMPLWI, cr, left, k);
1687         return;
1688       }
1689     }
1690   }
1691   right = ra_alloc1(as, rref, rset_exclude(RSET_GPR, left));
1692   emit_tab(as, (cc & CC_UNSIGNED) ? PPCI_CMPLW : PPCI_CMPW, cr, left, right);
1693 }
1694 
1695 static void asm_comp(ASMState *as, IRIns *ir)
1696 {
1697   PPCCC cc = asm_compmap[ir->o];
1698   if (irt_isnum(ir->t)) {
1699     Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1700     right = (left >> 8); left &= 255;
1701     asm_guardcc(as, (cc >> 4));
1702     if ((cc & CC_TWO))
1703       emit_tab(as, PPCI_CROR, ((cc>>4)&3), ((cc>>4)&3), (CC_EQ&3));
1704     emit_fab(as, PPCI_FCMPU, 0, left, right);
1705   } else {
1706     IRRef lref = ir->op1, rref = ir->op2;
1707     if (irref_isk(lref) && !irref_isk(rref)) {
1708       /* Swap constants to the right (only for ABC). */
1709       IRRef tmp = lref; lref = rref; rref = tmp;
1710       if ((cc & 2) == 0) cc ^= 1;  /* LT <-> GT, LE <-> GE */
1711     }
1712     asm_guardcc(as, cc);
1713     asm_intcomp_(as, lref, rref, 0, cc);
1714   }
1715 }
1716 
1717 #if LJ_HASFFI
1718 /* 64 bit integer comparisons. */
1719 static void asm_comp64(ASMState *as, IRIns *ir)
1720 {
1721   PPCCC cc = asm_compmap[(ir-1)->o];
1722   if ((cc&3) == (CC_EQ&3)) {
1723     asm_guardcc(as, cc);
1724     emit_tab(as, (cc&4) ? PPCI_CRAND : PPCI_CROR,
1725              (CC_EQ&3), (CC_EQ&3), 4+(CC_EQ&3));
1726   } else {
1727     asm_guardcc(as, CC_EQ);
1728     emit_tab(as, PPCI_CROR, (CC_EQ&3), (CC_EQ&3), ((cc^~(cc>>2))&1));
1729     emit_tab(as, (cc&4) ? PPCI_CRAND : PPCI_CRANDC,
1730              (CC_EQ&3), (CC_EQ&3), 4+(cc&3));
1731   }
1732   /* Loword comparison sets cr1 and is unsigned, except for equality. */
1733   asm_intcomp_(as, (ir-1)->op1, (ir-1)->op2, 4,
1734                cc | ((cc&3) == (CC_EQ&3) ? 0 : CC_UNSIGNED));
1735   /* Hiword comparison sets cr0. */
1736   asm_intcomp_(as, ir->op1, ir->op2, 0, cc);
1737   as->flagmcp = NULL;  /* Doesn't work here. */
1738 }
1739 #endif
1740 
1741 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
1742 
1743 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
1744 static void asm_hiop(ASMState *as, IRIns *ir)
1745 {
1746 #if LJ_HASFFI
1747   /* HIOP is marked as a store because it needs its own DCE logic. */
1748   int uselo = ra_used(ir-1), usehi = ra_used(ir);  /* Loword/hiword used? */
1749   if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
1750   if ((ir-1)->o == IR_CONV) {  /* Conversions to/from 64 bit. */
1751     as->curins--;  /* Always skip the CONV. */
1752     if (usehi || uselo)
1753       asm_conv64(as, ir);
1754     return;
1755   } else if ((ir-1)->o <= IR_NE) {  /* 64 bit integer comparisons. ORDER IR. */
1756     as->curins--;  /* Always skip the loword comparison. */
1757     asm_comp64(as, ir);
1758     return;
1759   } else if ((ir-1)->o == IR_XSTORE) {
1760     as->curins--;  /* Handle both stores here. */
1761     if ((ir-1)->r != RID_SINK) {
1762       asm_xstore(as, ir, 0);
1763       asm_xstore(as, ir-1, 4);
1764     }
1765     return;
1766   }
1767   if (!usehi) return;  /* Skip unused hiword op for all remaining ops. */
1768   switch ((ir-1)->o) {
1769   case IR_ADD: as->curins--; asm_add64(as, ir); break;
1770   case IR_SUB: as->curins--; asm_sub64(as, ir); break;
1771   case IR_NEG: as->curins--; asm_neg64(as, ir); break;
1772   case IR_CALLN:
1773   case IR_CALLXS:
1774     if (!uselo)
1775       ra_allocref(as, ir->op1, RID2RSET(RID_RETLO));  /* Mark lo op as used. */
1776     break;
1777   case IR_CNEWI:
1778     /* Nothing to do here. Handled by lo op itself. */
1779     break;
1780   default: lua_assert(0); break;
1781   }
1782 #else
1783   UNUSED(as); UNUSED(ir); lua_assert(0);  /* Unused without FFI. */
1784 #endif
1785 }
1786 
1787 /* -- Stack handling ------------------------------------------------------ */
1788 
1789 /* Check Lua stack size for overflow. Use exit handler as fallback. */
1790 static void asm_stack_check(ASMState *as, BCReg topslot,
1791                             IRIns *irp, RegSet allow, ExitNo exitno)
1792 {
1793   /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
1794   Reg tmp, pbase = irp ? (ra_hasreg(irp->r) ? irp->r : RID_TMP) : RID_BASE;
1795   rset_clear(allow, pbase);
1796   tmp = allow ? rset_pickbot(allow) :
1797                 (pbase == RID_RETHI ? RID_RETLO : RID_RETHI);
1798   emit_condbranch(as, PPCI_BC, CC_LT, asm_exitstub_addr(as, exitno));
1799   if (allow == RSET_EMPTY)  /* Restore temp. register. */
1800     emit_tai(as, PPCI_LWZ, tmp, RID_SP, SPOFS_TMPW);
1801   else
1802     ra_modified(as, tmp);
1803   emit_ai(as, PPCI_CMPLWI, RID_TMP, (int32_t)(8*topslot));
1804   emit_tab(as, PPCI_SUBF, RID_TMP, pbase, tmp);
1805   emit_tai(as, PPCI_LWZ, tmp, tmp, offsetof(lua_State, maxstack));
1806   if (pbase == RID_TMP)
1807     emit_getgl(as, RID_TMP, jit_base);
1808   emit_getgl(as, tmp, jit_L);
1809   if (allow == RSET_EMPTY)  /* Spill temp. register. */
1810     emit_tai(as, PPCI_STW, tmp, RID_SP, SPOFS_TMPW);
1811 }
1812 
1813 /* Restore Lua stack from on-trace state. */
1814 static void asm_stack_restore(ASMState *as, SnapShot *snap)
1815 {
1816   SnapEntry *map = &as->T->snapmap[snap->mapofs];
1817   SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1];
1818   MSize n, nent = snap->nent;
1819   /* Store the value of all modified slots to the Lua stack. */
1820   for (n = 0; n < nent; n++) {
1821     SnapEntry sn = map[n];
1822     BCReg s = snap_slot(sn);
1823     int32_t ofs = 8*((int32_t)s-1);
1824     IRRef ref = snap_ref(sn);
1825     IRIns *ir = IR(ref);
1826     if ((sn & SNAP_NORESTORE))
1827       continue;
1828     if (irt_isnum(ir->t)) {
1829       Reg src = ra_alloc1(as, ref, RSET_FPR);
1830       emit_fai(as, PPCI_STFD, src, RID_BASE, ofs);
1831     } else {
1832       Reg type;
1833       RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
1834       lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t));
1835       if (!irt_ispri(ir->t)) {
1836         Reg src = ra_alloc1(as, ref, allow);
1837         rset_clear(allow, src);
1838         emit_tai(as, PPCI_STW, src, RID_BASE, ofs+4);
1839       }
1840       if ((sn & (SNAP_CONT|SNAP_FRAME))) {
1841         if (s == 0) continue;  /* Do not overwrite link to previous frame. */
1842         type = ra_allock(as, (int32_t)(*flinks--), allow);
1843       } else {
1844         type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
1845       }
1846       emit_tai(as, PPCI_STW, type, RID_BASE, ofs);
1847     }
1848     checkmclim(as);
1849   }
1850   lua_assert(map + nent == flinks);
1851 }
1852 
1853 /* -- GC handling --------------------------------------------------------- */
1854 
1855 /* Check GC threshold and do one or more GC steps. */
1856 static void asm_gc_check(ASMState *as)
1857 {
1858   const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_step_jit];
1859   IRRef args[2];
1860   MCLabel l_end;
1861   Reg tmp;
1862   ra_evictset(as, RSET_SCRATCH);
1863   l_end = emit_label(as);
1864   /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
1865   asm_guardcc(as, CC_NE);  /* Assumes asm_snap_prep() already done. */
1866   emit_ai(as, PPCI_CMPWI, RID_RET, 0);
1867   args[0] = ASMREF_TMP1;  /* global_State *g */
1868   args[1] = ASMREF_TMP2;  /* MSize steps     */
1869   asm_gencall(as, ci, args);
1870   emit_tai(as, PPCI_ADDI, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
1871   tmp = ra_releasetmp(as, ASMREF_TMP2);
1872   emit_loadi(as, tmp, as->gcsteps);
1873   /* Jump around GC step if GC total < GC threshold. */
1874   emit_condbranch(as, PPCI_BC|PPCF_Y, CC_LT, l_end);
1875   emit_ab(as, PPCI_CMPLW, RID_TMP, tmp);
1876   emit_getgl(as, tmp, gc.threshold);
1877   emit_getgl(as, RID_TMP, gc.total);
1878   as->gcsteps = 0;
1879   checkmclim(as);
1880 }
1881 
1882 /* -- Loop handling ------------------------------------------------------- */
1883 
1884 /* Fixup the loop branch. */
1885 static void asm_loop_fixup(ASMState *as)
1886 {
1887   MCode *p = as->mctop;
1888   MCode *target = as->mcp;
1889   if (as->loopinv) {  /* Inverted loop branch? */
1890     /* asm_guardcc already inverted the cond branch and patched the final b. */
1891     p[-2] = (p[-2] & (0xffff0000u & ~PPCF_Y)) | (((target-p+2) & 0x3fffu) << 2);
1892   } else {
1893     p[-1] = PPCI_B|(((target-p+1)&0x00ffffffu)<<2);
1894   }
1895 }
1896 
1897 /* -- Head of trace ------------------------------------------------------- */
1898 
1899 /* Coalesce BASE register for a root trace. */
1900 static void asm_head_root_base(ASMState *as)
1901 {
1902   IRIns *ir = IR(REF_BASE);
1903   Reg r = ir->r;
1904   if (ra_hasreg(r)) {
1905     ra_free(as, r);
1906     if (rset_test(as->modset, r) || irt_ismarked(ir->t))
1907       ir->r = RID_INIT;  /* No inheritance for modified BASE register. */
1908     if (r != RID_BASE)
1909       emit_mr(as, r, RID_BASE);
1910   }
1911 }
1912 
1913 /* Coalesce BASE register for a side trace. */
1914 static RegSet asm_head_side_base(ASMState *as, IRIns *irp, RegSet allow)
1915 {
1916   IRIns *ir = IR(REF_BASE);
1917   Reg r = ir->r;
1918   if (ra_hasreg(r)) {
1919     ra_free(as, r);
1920     if (rset_test(as->modset, r) || irt_ismarked(ir->t))
1921       ir->r = RID_INIT;  /* No inheritance for modified BASE register. */
1922     if (irp->r == r) {
1923       rset_clear(allow, r);  /* Mark same BASE register as coalesced. */
1924     } else if (ra_hasreg(irp->r) && rset_test(as->freeset, irp->r)) {
1925       rset_clear(allow, irp->r);
1926       emit_mr(as, r, irp->r);  /* Move from coalesced parent reg. */
1927     } else {
1928       emit_getgl(as, r, jit_base);  /* Otherwise reload BASE. */
1929     }
1930   }
1931   return allow;
1932 }
1933 
1934 /* -- Tail of trace ------------------------------------------------------- */
1935 
1936 /* Fixup the tail code. */
1937 static void asm_tail_fixup(ASMState *as, TraceNo lnk)
1938 {
1939   MCode *p = as->mctop;
1940   MCode *target;
1941   int32_t spadj = as->T->spadjust;
1942   if (spadj == 0) {
1943     *--p = PPCI_NOP;
1944     *--p = PPCI_NOP;
1945     as->mctop = p;
1946   } else {
1947     /* Patch stack adjustment. */
1948     lua_assert(checki16(CFRAME_SIZE+spadj));
1949     p[-3] = PPCI_ADDI | PPCF_T(RID_TMP) | PPCF_A(RID_SP) | (CFRAME_SIZE+spadj);
1950     p[-2] = PPCI_STWU | PPCF_T(RID_TMP) | PPCF_A(RID_SP) | spadj;
1951   }
1952   /* Patch exit branch. */
1953   target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp;
1954   p[-1] = PPCI_B|(((target-p+1)&0x00ffffffu)<<2);
1955 }
1956 
1957 /* Prepare tail of code. */
1958 static void asm_tail_prep(ASMState *as)
1959 {
1960   MCode *p = as->mctop - 1;  /* Leave room for exit branch. */
1961   if (as->loopref) {
1962     as->invmcp = as->mcp = p;
1963   } else {
1964     as->mcp = p-2;  /* Leave room for stack pointer adjustment. */
1965     as->invmcp = NULL;
1966   }
1967 }
1968 
1969 /* -- Instruction dispatch ------------------------------------------------ */
1970 
1971 /* Assemble a single instruction. */
1972 static void asm_ir(ASMState *as, IRIns *ir)
1973 {
1974   switch ((IROp)ir->o) {
1975   /* Miscellaneous ops. */
1976   case IR_LOOP: asm_loop(as); break;
1977   case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break;
1978   case IR_USE:
1979     ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
1980   case IR_PHI: asm_phi(as, ir); break;
1981   case IR_HIOP: asm_hiop(as, ir); break;
1982   case IR_GCSTEP: asm_gcstep(as, ir); break;
1983 
1984   /* Guarded assertions. */
1985   case IR_EQ: case IR_NE:
1986     if ((ir-1)->o == IR_HREF && ir->op1 == as->curins-1) {
1987       as->curins--;
1988       asm_href(as, ir-1, (IROp)ir->o);
1989       break;
1990     }
1991     /* fallthrough */
1992   case IR_LT: case IR_GE: case IR_LE: case IR_GT:
1993   case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
1994   case IR_ABC:
1995     asm_comp(as, ir);
1996     break;
1997 
1998   case IR_RETF: asm_retf(as, ir); break;
1999 
2000   /* Bit ops. */
2001   case IR_BNOT: asm_bitnot(as, ir); break;
2002   case IR_BSWAP: asm_bitswap(as, ir); break;
2003 
2004   case IR_BAND: asm_bitand(as, ir); break;
2005   case IR_BOR:  asm_bitop(as, ir, PPCI_OR, PPCI_ORI); break;
2006   case IR_BXOR: asm_bitop(as, ir, PPCI_XOR, PPCI_XORI); break;
2007 
2008   case IR_BSHL: asm_bitshift(as, ir, PPCI_SLW, 0); break;
2009   case IR_BSHR: asm_bitshift(as, ir, PPCI_SRW, 1); break;
2010   case IR_BSAR: asm_bitshift(as, ir, PPCI_SRAW, PPCI_SRAWI); break;
2011   case IR_BROL: asm_bitshift(as, ir, PPCI_RLWNM|PPCF_MB(0)|PPCF_ME(31),
2012                              PPCI_RLWINM|PPCF_MB(0)|PPCF_ME(31)); break;
2013   case IR_BROR: lua_assert(0); break;
2014 
2015   /* Arithmetic ops. */
2016   case IR_ADD: asm_add(as, ir); break;
2017   case IR_SUB: asm_sub(as, ir); break;
2018   case IR_MUL: asm_mul(as, ir); break;
2019   case IR_DIV: asm_fparith(as, ir, PPCI_FDIV); break;
2020   case IR_MOD: asm_callid(as, ir, IRCALL_lj_vm_modi); break;
2021   case IR_POW: asm_callid(as, ir, IRCALL_lj_vm_powi); break;
2022   case IR_NEG: asm_neg(as, ir); break;
2023 
2024   case IR_ABS: asm_fpunary(as, ir, PPCI_FABS); break;
2025   case IR_ATAN2: asm_callid(as, ir, IRCALL_atan2); break;
2026   case IR_LDEXP: asm_callid(as, ir, IRCALL_ldexp); break;
2027   case IR_MIN: asm_min_max(as, ir, 0); break;
2028   case IR_MAX: asm_min_max(as, ir, 1); break;
2029   case IR_FPMATH:
2030     if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
2031       break;
2032     if (ir->op2 == IRFPM_SQRT && (as->flags & JIT_F_SQRT))
2033       asm_fpunary(as, ir, PPCI_FSQRT);
2034     else
2035       asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
2036     break;
2037 
2038   /* Overflow-checking arithmetic ops. */
2039   case IR_ADDOV: asm_arithov(as, ir, PPCI_ADDO); break;
2040   case IR_SUBOV: asm_arithov(as, ir, PPCI_SUBFO); break;
2041   case IR_MULOV: asm_arithov(as, ir, PPCI_MULLWO); break;
2042 
2043   /* Memory references. */
2044   case IR_AREF: asm_aref(as, ir); break;
2045   case IR_HREF: asm_href(as, ir, 0); break;
2046   case IR_HREFK: asm_hrefk(as, ir); break;
2047   case IR_NEWREF: asm_newref(as, ir); break;
2048   case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
2049   case IR_FREF: asm_fref(as, ir); break;
2050   case IR_STRREF: asm_strref(as, ir); break;
2051 
2052   /* Loads and stores. */
2053   case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
2054     asm_ahuvload(as, ir);
2055     break;
2056   case IR_FLOAD: asm_fload(as, ir); break;
2057   case IR_XLOAD: asm_xload(as, ir); break;
2058   case IR_SLOAD: asm_sload(as, ir); break;
2059 
2060   case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
2061   case IR_FSTORE: asm_fstore(as, ir); break;
2062   case IR_XSTORE: asm_xstore(as, ir, 0); break;
2063 
2064   /* Allocations. */
2065   case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
2066   case IR_TNEW: asm_tnew(as, ir); break;
2067   case IR_TDUP: asm_tdup(as, ir); break;
2068   case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
2069 
2070   /* Write barriers. */
2071   case IR_TBAR: asm_tbar(as, ir); break;
2072   case IR_OBAR: asm_obar(as, ir); break;
2073 
2074   /* Type conversions. */
2075   case IR_CONV: asm_conv(as, ir); break;
2076   case IR_TOBIT: asm_tobit(as, ir); break;
2077   case IR_TOSTR: asm_tostr(as, ir); break;
2078   case IR_STRTO: asm_strto(as, ir); break;
2079 
2080   /* Calls. */
2081   case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
2082   case IR_CALLXS: asm_callx(as, ir); break;
2083   case IR_CARG: break;
2084 
2085   default:
2086     setintV(&as->J->errinfo, ir->o);
2087     lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
2088     break;
2089   }
2090 }
2091 
2092 /* -- Trace setup --------------------------------------------------------- */
2093 
2094 /* Ensure there are enough stack slots for call arguments. */
2095 static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
2096 {
2097   IRRef args[CCI_NARGS_MAX*2];
2098   uint32_t i, nargs = (int)CCI_NARGS(ci);
2099   int nslots = 2, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
2100   asm_collectargs(as, ir, ci, args);
2101   for (i = 0; i < nargs; i++)
2102     if (args[i] && irt_isfp(IR(args[i])->t)) {
2103       if (nfpr > 0) nfpr--; else nslots = (nslots+3) & ~1;
2104     } else {
2105       if (ngpr > 0) ngpr--; else nslots++;
2106     }
2107   if (nslots > as->evenspill)  /* Leave room for args in stack slots. */
2108     as->evenspill = nslots;
2109   return irt_isfp(ir->t) ? REGSP_HINT(RID_FPRET) : REGSP_HINT(RID_RET);
2110 }
2111 
2112 static void asm_setup_target(ASMState *as)
2113 {
2114   asm_exitstub_setup(as, as->T->nsnap + (as->parent ? 1 : 0));
2115 }
2116 
2117 /* -- Trace patching ------------------------------------------------------ */
2118 
2119 /* Patch exit jumps of existing machine code to a new target. */
2120 void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
2121 {
2122   MCode *p = T->mcode;
2123   MCode *pe = (MCode *)((char *)p + T->szmcode);
2124   MCode *px = exitstub_trace_addr(T, exitno);
2125   MCode *cstart = NULL;
2126   MCode *mcarea = lj_mcode_patch(J, p, 0);
2127   int clearso = 0;
2128   for (; p < pe; p++) {
2129     /* Look for exitstub branch, try to replace with branch to target. */
2130     uint32_t ins = *p;
2131     if ((ins & 0xfc000000u) == 0x40000000u &&
2132         ((ins ^ ((char *)px-(char *)p)) & 0xffffu) == 0) {
2133       ptrdiff_t delta = (char *)target - (char *)p;
2134       if (((ins >> 16) & 3) == (CC_SO&3)) {
2135         clearso = sizeof(MCode);
2136         delta -= sizeof(MCode);
2137       }
2138       /* Many, but not all short-range branches can be patched directly. */
2139       if (((delta + 0x8000) >> 16) == 0) {
2140         *p = (ins & 0xffdf0000u) | ((uint32_t)delta & 0xffffu) |
2141              ((delta & 0x8000) * (PPCF_Y/0x8000));
2142         if (!cstart) cstart = p;
2143       }
2144     } else if ((ins & 0xfc000000u) == PPCI_B &&
2145                ((ins ^ ((char *)px-(char *)p)) & 0x03ffffffu) == 0) {
2146       ptrdiff_t delta = (char *)target - (char *)p;
2147       lua_assert(((delta + 0x02000000) >> 26) == 0);
2148       *p = PPCI_B | ((uint32_t)delta & 0x03ffffffu);
2149       if (!cstart) cstart = p;
2150     }
2151   }
2152   {  /* Always patch long-range branch in exit stub itself. */
2153     ptrdiff_t delta = (char *)target - (char *)px - clearso;
2154     lua_assert(((delta + 0x02000000) >> 26) == 0);
2155     *px = PPCI_B | ((uint32_t)delta & 0x03ffffffu);
2156   }
2157   if (!cstart) cstart = px;
2158   lj_mcode_sync(cstart, px+1);
2159   if (clearso) {  /* Extend the current trace. Ugly workaround. */
2160     MCode *pp = J->cur.mcode;
2161     J->cur.szmcode += sizeof(MCode);
2162     *--pp = PPCI_MCRXR;  /* Clear SO flag. */
2163     J->cur.mcode = pp;
2164     lj_mcode_sync(pp, pp+1);
2165   }
2166   lj_mcode_patch(J, mcarea, 1);
2167 }
2168 

/* [<][>][^][v][top][bottom][index][help] */