root/lj_asm_x86.h

/* [<][>][^][v][top][bottom][index][help] */

INCLUDED FROM


DEFINITIONS

This source file includes following definitions.
  1. asm_exitstub_gen
  2. asm_exitstub_setup
  3. asm_guardcc
  4. asm_isk32
  5. noconflict
  6. asm_fuseabase
  7. asm_fusearef
  8. asm_fuseahuref
  9. asm_fusefref
  10. asm_fusestrref
  11. asm_fusexref
  12. asm_fuseloadk64
  13. asm_fuseload
  14. asm_fuseloadm
  15. asm_count_call_slots
  16. asm_gencall
  17. asm_setupresult
  18. asm_callx_func
  19. asm_callx
  20. asm_retf
  21. asm_tointg
  22. asm_tobit
  23. asm_conv
  24. asm_conv_fp_int64
  25. asm_conv_int64_fp
  26. asm_conv64
  27. asm_strto
  28. asm_tvptr
  29. asm_aref
  30. asm_href
  31. asm_hrefk
  32. asm_uref
  33. asm_fref
  34. asm_strref
  35. asm_fxload
  36. asm_fxstore
  37. asm_load_lightud64
  38. asm_ahuvload
  39. asm_ahustore
  40. asm_sload
  41. asm_cnew
  42. asm_tbar
  43. asm_obar
  44. asm_x87load
  45. asm_fpmath
  46. asm_ldexp
  47. asm_fppowi
  48. asm_pow
  49. asm_swapops
  50. asm_fparith
  51. asm_intarith
  52. asm_lea
  53. asm_add
  54. asm_sub
  55. asm_mul
  56. asm_div
  57. asm_mod
  58. asm_neg_not
  59. asm_neg
  60. asm_intmin_max
  61. asm_min
  62. asm_max
  63. asm_bswap
  64. asm_bitshift
  65. asm_comp
  66. asm_comp_int64
  67. asm_hiop
  68. asm_prof
  69. asm_stack_check
  70. asm_stack_restore
  71. asm_gc_check
  72. asm_loop_fixup
  73. asm_head_root_base
  74. asm_head_side_base
  75. asm_tail_fixup
  76. asm_tail_prep
  77. asm_setup_call_slots
  78. asm_setup_target
  79. asm_x86_inslen
  80. lj_asm_patchexit

   1 /*
   2 ** x86/x64 IR assembler (SSA IR -> machine code).
   3 ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5 
   6 /* -- Guard handling ------------------------------------------------------ */
   7 
   8 /* Generate an exit stub group at the bottom of the reserved MCode memory. */
   9 static MCode *asm_exitstub_gen(ASMState *as, ExitNo group)
  10 {
  11   ExitNo i, groupofs = (group*EXITSTUBS_PER_GROUP) & 0xff;
  12   MCode *mxp = as->mcbot;
  13   MCode *mxpstart = mxp;
  14   if (mxp + (2+2)*EXITSTUBS_PER_GROUP+8+5 >= as->mctop)
  15     asm_mclimit(as);
  16   /* Push low byte of exitno for each exit stub. */
  17   *mxp++ = XI_PUSHi8; *mxp++ = (MCode)groupofs;
  18   for (i = 1; i < EXITSTUBS_PER_GROUP; i++) {
  19     *mxp++ = XI_JMPs; *mxp++ = (MCode)((2+2)*(EXITSTUBS_PER_GROUP - i) - 2);
  20     *mxp++ = XI_PUSHi8; *mxp++ = (MCode)(groupofs + i);
  21   }
  22   /* Push the high byte of the exitno for each exit stub group. */
  23   *mxp++ = XI_PUSHi8; *mxp++ = (MCode)((group*EXITSTUBS_PER_GROUP)>>8);
  24 #if !LJ_GC64
  25   /* Store DISPATCH at original stack slot 0. Account for the two push ops. */
  26   *mxp++ = XI_MOVmi;
  27   *mxp++ = MODRM(XM_OFS8, 0, RID_ESP);
  28   *mxp++ = MODRM(XM_SCALE1, RID_ESP, RID_ESP);
  29   *mxp++ = 2*sizeof(void *);
  30   *(int32_t *)mxp = ptr2addr(J2GG(as->J)->dispatch); mxp += 4;
  31 #endif
  32   /* Jump to exit handler which fills in the ExitState. */
  33   *mxp++ = XI_JMP; mxp += 4;
  34   *((int32_t *)(mxp-4)) = jmprel(mxp, (MCode *)(void *)lj_vm_exit_handler);
  35   /* Commit the code for this group (even if assembly fails later on). */
  36   lj_mcode_commitbot(as->J, mxp);
  37   as->mcbot = mxp;
  38   as->mclim = as->mcbot + MCLIM_REDZONE;
  39   return mxpstart;
  40 }
  41 
  42 /* Setup all needed exit stubs. */
  43 static void asm_exitstub_setup(ASMState *as, ExitNo nexits)
  44 {
  45   ExitNo i;
  46   if (nexits >= EXITSTUBS_PER_GROUP*LJ_MAX_EXITSTUBGR)
  47     lj_trace_err(as->J, LJ_TRERR_SNAPOV);
  48   for (i = 0; i < (nexits+EXITSTUBS_PER_GROUP-1)/EXITSTUBS_PER_GROUP; i++)
  49     if (as->J->exitstubgroup[i] == NULL)
  50       as->J->exitstubgroup[i] = asm_exitstub_gen(as, i);
  51 }
  52 
  53 /* Emit conditional branch to exit for guard.
  54 ** It's important to emit this *after* all registers have been allocated,
  55 ** because rematerializations may invalidate the flags.
  56 */
  57 static void asm_guardcc(ASMState *as, int cc)
  58 {
  59   MCode *target = exitstub_addr(as->J, as->snapno);
  60   MCode *p = as->mcp;
  61   if (LJ_UNLIKELY(p == as->invmcp)) {
  62     as->loopinv = 1;
  63     *(int32_t *)(p+1) = jmprel(p+5, target);
  64     target = p;
  65     cc ^= 1;
  66     if (as->realign) {
  67       if (LJ_GC64 && LJ_UNLIKELY(as->mrm.base == RID_RIP))
  68         as->mrm.ofs += 2;  /* Fixup RIP offset for pending fused load. */
  69       emit_sjcc(as, cc, target);
  70       return;
  71     }
  72   }
  73   if (LJ_GC64 && LJ_UNLIKELY(as->mrm.base == RID_RIP))
  74     as->mrm.ofs += 6;  /* Fixup RIP offset for pending fused load. */
  75   emit_jcc(as, cc, target);
  76 }
  77 
  78 /* -- Memory operand fusion ----------------------------------------------- */
  79 
  80 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
  81 #define CONFLICT_SEARCH_LIM     31
  82 
  83 /* Check if a reference is a signed 32 bit constant. */
  84 static int asm_isk32(ASMState *as, IRRef ref, int32_t *k)
  85 {
  86   if (irref_isk(ref)) {
  87     IRIns *ir = IR(ref);
  88 #if LJ_GC64
  89     if (ir->o == IR_KNULL || !irt_is64(ir->t)) {
  90       *k = ir->i;
  91       return 1;
  92     } else if (checki32((int64_t)ir_k64(ir)->u64)) {
  93       *k = (int32_t)ir_k64(ir)->u64;
  94       return 1;
  95     }
  96 #else
  97     if (ir->o != IR_KINT64) {
  98       *k = ir->i;
  99       return 1;
 100     } else if (checki32((int64_t)ir_kint64(ir)->u64)) {
 101       *k = (int32_t)ir_kint64(ir)->u64;
 102       return 1;
 103     }
 104 #endif
 105   }
 106   return 0;
 107 }
 108 
 109 /* Check if there's no conflicting instruction between curins and ref.
 110 ** Also avoid fusing loads if there are multiple references.
 111 */
 112 static int noconflict(ASMState *as, IRRef ref, IROp conflict, int noload)
 113 {
 114   IRIns *ir = as->ir;
 115   IRRef i = as->curins;
 116   if (i > ref + CONFLICT_SEARCH_LIM)
 117     return 0;  /* Give up, ref is too far away. */
 118   while (--i > ref) {
 119     if (ir[i].o == conflict)
 120       return 0;  /* Conflict found. */
 121     else if (!noload && (ir[i].op1 == ref || ir[i].op2 == ref))
 122       return 0;
 123   }
 124   return 1;  /* Ok, no conflict. */
 125 }
 126 
 127 /* Fuse array base into memory operand. */
 128 static IRRef asm_fuseabase(ASMState *as, IRRef ref)
 129 {
 130   IRIns *irb = IR(ref);
 131   as->mrm.ofs = 0;
 132   if (irb->o == IR_FLOAD) {
 133     IRIns *ira = IR(irb->op1);
 134     lua_assert(irb->op2 == IRFL_TAB_ARRAY);
 135     /* We can avoid the FLOAD of t->array for colocated arrays. */
 136     if (ira->o == IR_TNEW && ira->op1 <= LJ_MAX_COLOSIZE &&
 137         !neverfuse(as) && noconflict(as, irb->op1, IR_NEWREF, 1)) {
 138       as->mrm.ofs = (int32_t)sizeof(GCtab);  /* Ofs to colocated array. */
 139       return irb->op1;  /* Table obj. */
 140     }
 141   } else if (irb->o == IR_ADD && irref_isk(irb->op2)) {
 142     /* Fuse base offset (vararg load). */
 143     as->mrm.ofs = IR(irb->op2)->i;
 144     return irb->op1;
 145   }
 146   return ref;  /* Otherwise use the given array base. */
 147 }
 148 
 149 /* Fuse array reference into memory operand. */
 150 static void asm_fusearef(ASMState *as, IRIns *ir, RegSet allow)
 151 {
 152   IRIns *irx;
 153   lua_assert(ir->o == IR_AREF);
 154   as->mrm.base = (uint8_t)ra_alloc1(as, asm_fuseabase(as, ir->op1), allow);
 155   irx = IR(ir->op2);
 156   if (irref_isk(ir->op2)) {
 157     as->mrm.ofs += 8*irx->i;
 158     as->mrm.idx = RID_NONE;
 159   } else {
 160     rset_clear(allow, as->mrm.base);
 161     as->mrm.scale = XM_SCALE8;
 162     /* Fuse a constant ADD (e.g. t[i+1]) into the offset.
 163     ** Doesn't help much without ABCelim, but reduces register pressure.
 164     */
 165     if (!LJ_64 &&  /* Has bad effects with negative index on x64. */
 166         mayfuse(as, ir->op2) && ra_noreg(irx->r) &&
 167         irx->o == IR_ADD && irref_isk(irx->op2)) {
 168       as->mrm.ofs += 8*IR(irx->op2)->i;
 169       as->mrm.idx = (uint8_t)ra_alloc1(as, irx->op1, allow);
 170     } else {
 171       as->mrm.idx = (uint8_t)ra_alloc1(as, ir->op2, allow);
 172     }
 173   }
 174 }
 175 
 176 /* Fuse array/hash/upvalue reference into memory operand.
 177 ** Caveat: this may allocate GPRs for the base/idx registers. Be sure to
 178 ** pass the final allow mask, excluding any GPRs used for other inputs.
 179 ** In particular: 2-operand GPR instructions need to call ra_dest() first!
 180 */
 181 static void asm_fuseahuref(ASMState *as, IRRef ref, RegSet allow)
 182 {
 183   IRIns *ir = IR(ref);
 184   if (ra_noreg(ir->r)) {
 185     switch ((IROp)ir->o) {
 186     case IR_AREF:
 187       if (mayfuse(as, ref)) {
 188         asm_fusearef(as, ir, allow);
 189         return;
 190       }
 191       break;
 192     case IR_HREFK:
 193       if (mayfuse(as, ref)) {
 194         as->mrm.base = (uint8_t)ra_alloc1(as, ir->op1, allow);
 195         as->mrm.ofs = (int32_t)(IR(ir->op2)->op2 * sizeof(Node));
 196         as->mrm.idx = RID_NONE;
 197         return;
 198       }
 199       break;
 200     case IR_UREFC:
 201       if (irref_isk(ir->op1)) {
 202         GCfunc *fn = ir_kfunc(IR(ir->op1));
 203         GCupval *uv = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv;
 204 #if LJ_GC64
 205         int64_t ofs = dispofs(as, &uv->tv);
 206         if (checki32(ofs) && checki32(ofs+4)) {
 207           as->mrm.ofs = (int32_t)ofs;
 208           as->mrm.base = RID_DISPATCH;
 209           as->mrm.idx = RID_NONE;
 210           return;
 211         }
 212 #else
 213         as->mrm.ofs = ptr2addr(&uv->tv);
 214         as->mrm.base = as->mrm.idx = RID_NONE;
 215         return;
 216 #endif
 217       }
 218       break;
 219     default:
 220       lua_assert(ir->o == IR_HREF || ir->o == IR_NEWREF || ir->o == IR_UREFO ||
 221                  ir->o == IR_KKPTR);
 222       break;
 223     }
 224   }
 225   as->mrm.base = (uint8_t)ra_alloc1(as, ref, allow);
 226   as->mrm.ofs = 0;
 227   as->mrm.idx = RID_NONE;
 228 }
 229 
 230 /* Fuse FLOAD/FREF reference into memory operand. */
 231 static void asm_fusefref(ASMState *as, IRIns *ir, RegSet allow)
 232 {
 233   lua_assert(ir->o == IR_FLOAD || ir->o == IR_FREF);
 234   as->mrm.idx = RID_NONE;
 235   if (ir->op1 == REF_NIL) {
 236 #if LJ_GC64
 237     as->mrm.ofs = (int32_t)(ir->op2 << 2) - GG_OFS(dispatch);
 238     as->mrm.base = RID_DISPATCH;
 239 #else
 240     as->mrm.ofs = (int32_t)(ir->op2 << 2) + ptr2addr(J2GG(as->J));
 241     as->mrm.base = RID_NONE;
 242 #endif
 243     return;
 244   }
 245   as->mrm.ofs = field_ofs[ir->op2];
 246   if (irref_isk(ir->op1)) {
 247     IRIns *op1 = IR(ir->op1);
 248 #if LJ_GC64
 249     if (ir->op1 == REF_NIL) {
 250       as->mrm.ofs -= GG_OFS(dispatch);
 251       as->mrm.base = RID_DISPATCH;
 252       return;
 253     } else if (op1->o == IR_KPTR || op1->o == IR_KKPTR) {
 254       intptr_t ofs = dispofs(as, ir_kptr(op1));
 255       if (checki32(as->mrm.ofs + ofs)) {
 256         as->mrm.ofs += (int32_t)ofs;
 257         as->mrm.base = RID_DISPATCH;
 258         return;
 259       }
 260     }
 261 #else
 262     as->mrm.ofs += op1->i;
 263     as->mrm.base = RID_NONE;
 264     return;
 265 #endif
 266   }
 267   as->mrm.base = (uint8_t)ra_alloc1(as, ir->op1, allow);
 268 }
 269 
 270 /* Fuse string reference into memory operand. */
 271 static void asm_fusestrref(ASMState *as, IRIns *ir, RegSet allow)
 272 {
 273   IRIns *irr;
 274   lua_assert(ir->o == IR_STRREF);
 275   as->mrm.base = as->mrm.idx = RID_NONE;
 276   as->mrm.scale = XM_SCALE1;
 277   as->mrm.ofs = sizeof(GCstr);
 278   if (!LJ_GC64 && irref_isk(ir->op1)) {
 279     as->mrm.ofs += IR(ir->op1)->i;
 280   } else {
 281     Reg r = ra_alloc1(as, ir->op1, allow);
 282     rset_clear(allow, r);
 283     as->mrm.base = (uint8_t)r;
 284   }
 285   irr = IR(ir->op2);
 286   if (irref_isk(ir->op2)) {
 287     as->mrm.ofs += irr->i;
 288   } else {
 289     Reg r;
 290     /* Fuse a constant add into the offset, e.g. string.sub(s, i+10). */
 291     if (!LJ_64 &&  /* Has bad effects with negative index on x64. */
 292         mayfuse(as, ir->op2) && irr->o == IR_ADD && irref_isk(irr->op2)) {
 293       as->mrm.ofs += IR(irr->op2)->i;
 294       r = ra_alloc1(as, irr->op1, allow);
 295     } else {
 296       r = ra_alloc1(as, ir->op2, allow);
 297     }
 298     if (as->mrm.base == RID_NONE)
 299       as->mrm.base = (uint8_t)r;
 300     else
 301       as->mrm.idx = (uint8_t)r;
 302   }
 303 }
 304 
 305 static void asm_fusexref(ASMState *as, IRRef ref, RegSet allow)
 306 {
 307   IRIns *ir = IR(ref);
 308   as->mrm.idx = RID_NONE;
 309   if (ir->o == IR_KPTR || ir->o == IR_KKPTR) {
 310 #if LJ_GC64
 311     intptr_t ofs = dispofs(as, ir_kptr(ir));
 312     if (checki32(ofs)) {
 313       as->mrm.ofs = (int32_t)ofs;
 314       as->mrm.base = RID_DISPATCH;
 315       return;
 316     }
 317   } if (0) {
 318 #else
 319     as->mrm.ofs = ir->i;
 320     as->mrm.base = RID_NONE;
 321   } else if (ir->o == IR_STRREF) {
 322     asm_fusestrref(as, ir, allow);
 323 #endif
 324   } else {
 325     as->mrm.ofs = 0;
 326     if (canfuse(as, ir) && ir->o == IR_ADD && ra_noreg(ir->r)) {
 327       /* Gather (base+idx*sz)+ofs as emitted by cdata ptr/array indexing. */
 328       IRIns *irx;
 329       IRRef idx;
 330       Reg r;
 331       if (asm_isk32(as, ir->op2, &as->mrm.ofs)) {  /* Recognize x+ofs. */
 332         ref = ir->op1;
 333         ir = IR(ref);
 334         if (!(ir->o == IR_ADD && canfuse(as, ir) && ra_noreg(ir->r)))
 335           goto noadd;
 336       }
 337       as->mrm.scale = XM_SCALE1;
 338       idx = ir->op1;
 339       ref = ir->op2;
 340       irx = IR(idx);
 341       if (!(irx->o == IR_BSHL || irx->o == IR_ADD)) {  /* Try other operand. */
 342         idx = ir->op2;
 343         ref = ir->op1;
 344         irx = IR(idx);
 345       }
 346       if (canfuse(as, irx) && ra_noreg(irx->r)) {
 347         if (irx->o == IR_BSHL && irref_isk(irx->op2) && IR(irx->op2)->i <= 3) {
 348           /* Recognize idx<<b with b = 0-3, corresponding to sz = (1),2,4,8. */
 349           idx = irx->op1;
 350           as->mrm.scale = (uint8_t)(IR(irx->op2)->i << 6);
 351         } else if (irx->o == IR_ADD && irx->op1 == irx->op2) {
 352           /* FOLD does idx*2 ==> idx<<1 ==> idx+idx. */
 353           idx = irx->op1;
 354           as->mrm.scale = XM_SCALE2;
 355         }
 356       }
 357       r = ra_alloc1(as, idx, allow);
 358       rset_clear(allow, r);
 359       as->mrm.idx = (uint8_t)r;
 360     }
 361   noadd:
 362     as->mrm.base = (uint8_t)ra_alloc1(as, ref, allow);
 363   }
 364 }
 365 
 366 /* Fuse load of 64 bit IR constant into memory operand. */
 367 static Reg asm_fuseloadk64(ASMState *as, IRIns *ir)
 368 {
 369   const uint64_t *k = &ir_k64(ir)->u64;
 370   if (!LJ_GC64 || checki32((intptr_t)k)) {
 371     as->mrm.ofs = ptr2addr(k);
 372     as->mrm.base = RID_NONE;
 373 #if LJ_GC64
 374   } else if (checki32(dispofs(as, k))) {
 375     as->mrm.ofs = (int32_t)dispofs(as, k);
 376     as->mrm.base = RID_DISPATCH;
 377   } else if (checki32(mcpofs(as, k)) && checki32(mcpofs(as, k+1)) &&
 378              checki32(mctopofs(as, k)) && checki32(mctopofs(as, k+1))) {
 379     as->mrm.ofs = (int32_t)mcpofs(as, k);
 380     as->mrm.base = RID_RIP;
 381   } else {
 382     if (ir->i) {
 383       lua_assert(*k == *(uint64_t*)(as->mctop - ir->i));
 384     } else {
 385       while ((uintptr_t)as->mcbot & 7) *as->mcbot++ = XI_INT3;
 386       *(uint64_t*)as->mcbot = *k;
 387       ir->i = (int32_t)(as->mctop - as->mcbot);
 388       as->mcbot += 8;
 389       as->mclim = as->mcbot + MCLIM_REDZONE;
 390       lj_mcode_commitbot(as->J, as->mcbot);
 391     }
 392     as->mrm.ofs = (int32_t)mcpofs(as, as->mctop - ir->i);
 393     as->mrm.base = RID_RIP;
 394 #endif
 395   }
 396   as->mrm.idx = RID_NONE;
 397   return RID_MRM;
 398 }
 399 
 400 /* Fuse load into memory operand.
 401 **
 402 ** Important caveat: this may emit RIP-relative loads! So don't place any
 403 ** code emitters between this function and the use of its result.
 404 ** The only permitted exception is asm_guardcc().
 405 */
 406 static Reg asm_fuseload(ASMState *as, IRRef ref, RegSet allow)
 407 {
 408   IRIns *ir = IR(ref);
 409   if (ra_hasreg(ir->r)) {
 410     if (allow != RSET_EMPTY) {  /* Fast path. */
 411       ra_noweak(as, ir->r);
 412       return ir->r;
 413     }
 414   fusespill:
 415     /* Force a spill if only memory operands are allowed (asm_x87load). */
 416     as->mrm.base = RID_ESP;
 417     as->mrm.ofs = ra_spill(as, ir);
 418     as->mrm.idx = RID_NONE;
 419     return RID_MRM;
 420   }
 421   if (ir->o == IR_KNUM) {
 422     RegSet avail = as->freeset & ~as->modset & RSET_FPR;
 423     lua_assert(allow != RSET_EMPTY);
 424     if (!(avail & (avail-1)))  /* Fuse if less than two regs available. */
 425       return asm_fuseloadk64(as, ir);
 426   } else if (ref == REF_BASE || ir->o == IR_KINT64) {
 427     RegSet avail = as->freeset & ~as->modset & RSET_GPR;
 428     lua_assert(allow != RSET_EMPTY);
 429     if (!(avail & (avail-1))) {  /* Fuse if less than two regs available. */
 430       if (ref == REF_BASE) {
 431 #if LJ_GC64
 432         as->mrm.ofs = (int32_t)dispofs(as, &J2G(as->J)->jit_base);
 433         as->mrm.base = RID_DISPATCH;
 434 #else
 435         as->mrm.ofs = ptr2addr(&J2G(as->J)->jit_base);
 436         as->mrm.base = RID_NONE;
 437 #endif
 438         as->mrm.idx = RID_NONE;
 439         return RID_MRM;
 440       } else {
 441         return asm_fuseloadk64(as, ir);
 442       }
 443     }
 444   } else if (mayfuse(as, ref)) {
 445     RegSet xallow = (allow & RSET_GPR) ? allow : RSET_GPR;
 446     if (ir->o == IR_SLOAD) {
 447       if (!(ir->op2 & (IRSLOAD_PARENT|IRSLOAD_CONVERT)) &&
 448           noconflict(as, ref, IR_RETF, 0) &&
 449           !(LJ_GC64 && irt_isaddr(ir->t))) {
 450         as->mrm.base = (uint8_t)ra_alloc1(as, REF_BASE, xallow);
 451         as->mrm.ofs = 8*((int32_t)ir->op1-1-LJ_FR2) +
 452                       (!LJ_FR2 && (ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
 453         as->mrm.idx = RID_NONE;
 454         return RID_MRM;
 455       }
 456     } else if (ir->o == IR_FLOAD) {
 457       /* Generic fusion is only ok for 32 bit operand (but see asm_comp). */
 458       if ((irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t)) &&
 459           noconflict(as, ref, IR_FSTORE, 0)) {
 460         asm_fusefref(as, ir, xallow);
 461         return RID_MRM;
 462       }
 463     } else if (ir->o == IR_ALOAD || ir->o == IR_HLOAD || ir->o == IR_ULOAD) {
 464       if (noconflict(as, ref, ir->o + IRDELTA_L2S, 0) &&
 465           !(LJ_GC64 && irt_isaddr(ir->t))) {
 466         asm_fuseahuref(as, ir->op1, xallow);
 467         return RID_MRM;
 468       }
 469     } else if (ir->o == IR_XLOAD) {
 470       /* Generic fusion is not ok for 8/16 bit operands (but see asm_comp).
 471       ** Fusing unaligned memory operands is ok on x86 (except for SIMD types).
 472       */
 473       if ((!irt_typerange(ir->t, IRT_I8, IRT_U16)) &&
 474           noconflict(as, ref, IR_XSTORE, 0)) {
 475         asm_fusexref(as, ir->op1, xallow);
 476         return RID_MRM;
 477       }
 478     } else if (ir->o == IR_VLOAD && !(LJ_GC64 && irt_isaddr(ir->t))) {
 479       asm_fuseahuref(as, ir->op1, xallow);
 480       return RID_MRM;
 481     }
 482   }
 483   if (ir->o == IR_FLOAD && ir->op1 == REF_NIL) {
 484     asm_fusefref(as, ir, RSET_EMPTY);
 485     return RID_MRM;
 486   }
 487   if (!(as->freeset & allow) && !emit_canremat(ref) &&
 488       (allow == RSET_EMPTY || ra_hasspill(ir->s) || iscrossref(as, ref)))
 489     goto fusespill;
 490   return ra_allocref(as, ref, allow);
 491 }
 492 
 493 #if LJ_64
 494 /* Don't fuse a 32 bit load into a 64 bit operation. */
 495 static Reg asm_fuseloadm(ASMState *as, IRRef ref, RegSet allow, int is64)
 496 {
 497   if (is64 && !irt_is64(IR(ref)->t))
 498     return ra_alloc1(as, ref, allow);
 499   return asm_fuseload(as, ref, allow);
 500 }
 501 #else
 502 #define asm_fuseloadm(as, ref, allow, is64)  asm_fuseload(as, (ref), (allow))
 503 #endif
 504 
 505 /* -- Calls --------------------------------------------------------------- */
 506 
 507 /* Count the required number of stack slots for a call. */
 508 static int asm_count_call_slots(ASMState *as, const CCallInfo *ci, IRRef *args)
 509 {
 510   uint32_t i, nargs = CCI_XNARGS(ci);
 511   int nslots = 0;
 512 #if LJ_64
 513   if (LJ_ABI_WIN) {
 514     nslots = (int)(nargs*2);  /* Only matters for more than four args. */
 515   } else {
 516     int ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
 517     for (i = 0; i < nargs; i++)
 518       if (args[i] && irt_isfp(IR(args[i])->t)) {
 519         if (nfpr > 0) nfpr--; else nslots += 2;
 520       } else {
 521         if (ngpr > 0) ngpr--; else nslots += 2;
 522       }
 523   }
 524 #else
 525   int ngpr = 0;
 526   if ((ci->flags & CCI_CC_MASK) == CCI_CC_FASTCALL)
 527     ngpr = 2;
 528   else if ((ci->flags & CCI_CC_MASK) == CCI_CC_THISCALL)
 529     ngpr = 1;
 530   for (i = 0; i < nargs; i++)
 531     if (args[i] && irt_isfp(IR(args[i])->t)) {
 532       nslots += irt_isnum(IR(args[i])->t) ? 2 : 1;
 533     } else {
 534       if (ngpr > 0) ngpr--; else nslots++;
 535     }
 536 #endif
 537   return nslots;
 538 }
 539 
 540 /* Generate a call to a C function. */
 541 static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
 542 {
 543   uint32_t n, nargs = CCI_XNARGS(ci);
 544   int32_t ofs = STACKARG_OFS;
 545 #if LJ_64
 546   uint32_t gprs = REGARG_GPRS;
 547   Reg fpr = REGARG_FIRSTFPR;
 548 #if !LJ_ABI_WIN
 549   MCode *patchnfpr = NULL;
 550 #endif
 551 #else
 552   uint32_t gprs = 0;
 553   if ((ci->flags & CCI_CC_MASK) != CCI_CC_CDECL) {
 554     if ((ci->flags & CCI_CC_MASK) == CCI_CC_THISCALL)
 555       gprs = (REGARG_GPRS & 31);
 556     else if ((ci->flags & CCI_CC_MASK) == CCI_CC_FASTCALL)
 557       gprs = REGARG_GPRS;
 558   }
 559 #endif
 560   if ((void *)ci->func)
 561     emit_call(as, ci->func);
 562 #if LJ_64
 563   if ((ci->flags & CCI_VARARG)) {  /* Special handling for vararg calls. */
 564 #if LJ_ABI_WIN
 565     for (n = 0; n < 4 && n < nargs; n++) {
 566       IRIns *ir = IR(args[n]);
 567       if (irt_isfp(ir->t))  /* Duplicate FPRs in GPRs. */
 568         emit_rr(as, XO_MOVDto, (irt_isnum(ir->t) ? REX_64 : 0) | (fpr+n),
 569                 ((gprs >> (n*5)) & 31));  /* Either MOVD or MOVQ. */
 570     }
 571 #else
 572     patchnfpr = --as->mcp;  /* Indicate number of used FPRs in register al. */
 573     *--as->mcp = XI_MOVrib | RID_EAX;
 574 #endif
 575   }
 576 #endif
 577   for (n = 0; n < nargs; n++) {  /* Setup args. */
 578     IRRef ref = args[n];
 579     IRIns *ir = IR(ref);
 580     Reg r;
 581 #if LJ_64 && LJ_ABI_WIN
 582     /* Windows/x64 argument registers are strictly positional. */
 583     r = irt_isfp(ir->t) ? (fpr <= REGARG_LASTFPR ? fpr : 0) : (gprs & 31);
 584     fpr++; gprs >>= 5;
 585 #elif LJ_64
 586     /* POSIX/x64 argument registers are used in order of appearance. */
 587     if (irt_isfp(ir->t)) {
 588       r = fpr <= REGARG_LASTFPR ? fpr++ : 0;
 589     } else {
 590       r = gprs & 31; gprs >>= 5;
 591     }
 592 #else
 593     if (ref && irt_isfp(ir->t)) {
 594       r = 0;
 595     } else {
 596       r = gprs & 31; gprs >>= 5;
 597       if (!ref) continue;
 598     }
 599 #endif
 600     if (r) {  /* Argument is in a register. */
 601       if (r < RID_MAX_GPR && ref < ASMREF_TMP1) {
 602 #if LJ_64
 603         if (LJ_GC64 ? !(ir->o == IR_KINT || ir->o == IR_KNULL) : ir->o == IR_KINT64)
 604           emit_loadu64(as, r, ir_k64(ir)->u64);
 605         else
 606 #endif
 607           emit_loadi(as, r, ir->i);
 608       } else {
 609         lua_assert(rset_test(as->freeset, r));  /* Must have been evicted. */
 610         if (ra_hasreg(ir->r)) {
 611           ra_noweak(as, ir->r);
 612           emit_movrr(as, ir, r, ir->r);
 613         } else {
 614           ra_allocref(as, ref, RID2RSET(r));
 615         }
 616       }
 617     } else if (irt_isfp(ir->t)) {  /* FP argument is on stack. */
 618       lua_assert(!(irt_isfloat(ir->t) && irref_isk(ref)));  /* No float k. */
 619       if (LJ_32 && (ofs & 4) && irref_isk(ref)) {
 620         /* Split stores for unaligned FP consts. */
 621         emit_movmroi(as, RID_ESP, ofs, (int32_t)ir_knum(ir)->u32.lo);
 622         emit_movmroi(as, RID_ESP, ofs+4, (int32_t)ir_knum(ir)->u32.hi);
 623       } else {
 624         r = ra_alloc1(as, ref, RSET_FPR);
 625         emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSDto : XO_MOVSSto,
 626                   r, RID_ESP, ofs);
 627       }
 628       ofs += (LJ_32 && irt_isfloat(ir->t)) ? 4 : 8;
 629     } else {  /* Non-FP argument is on stack. */
 630       if (LJ_32 && ref < ASMREF_TMP1) {
 631         emit_movmroi(as, RID_ESP, ofs, ir->i);
 632       } else {
 633         r = ra_alloc1(as, ref, RSET_GPR);
 634         emit_movtomro(as, REX_64 + r, RID_ESP, ofs);
 635       }
 636       ofs += sizeof(intptr_t);
 637     }
 638     checkmclim(as);
 639   }
 640 #if LJ_64 && !LJ_ABI_WIN
 641   if (patchnfpr) *patchnfpr = fpr - REGARG_FIRSTFPR;
 642 #endif
 643 }
 644 
 645 /* Setup result reg/sp for call. Evict scratch regs. */
 646 static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
 647 {
 648   RegSet drop = RSET_SCRATCH;
 649   int hiop = (LJ_32 && (ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t));
 650   if ((ci->flags & CCI_NOFPRCLOBBER))
 651     drop &= ~RSET_FPR;
 652   if (ra_hasreg(ir->r))
 653     rset_clear(drop, ir->r);  /* Dest reg handled below. */
 654   if (hiop && ra_hasreg((ir+1)->r))
 655     rset_clear(drop, (ir+1)->r);  /* Dest reg handled below. */
 656   ra_evictset(as, drop);  /* Evictions must be performed first. */
 657   if (ra_used(ir)) {
 658     if (irt_isfp(ir->t)) {
 659       int32_t ofs = sps_scale(ir->s);  /* Use spill slot or temp slots. */
 660 #if LJ_64
 661       if ((ci->flags & CCI_CASTU64)) {
 662         Reg dest = ir->r;
 663         if (ra_hasreg(dest)) {
 664           ra_free(as, dest);
 665           ra_modified(as, dest);
 666           emit_rr(as, XO_MOVD, dest|REX_64, RID_RET);  /* Really MOVQ. */
 667         }
 668         if (ofs) emit_movtomro(as, RID_RET|REX_64, RID_ESP, ofs);
 669       } else {
 670         ra_destreg(as, ir, RID_FPRET);
 671       }
 672 #else
 673       /* Number result is in x87 st0 for x86 calling convention. */
 674       Reg dest = ir->r;
 675       if (ra_hasreg(dest)) {
 676         ra_free(as, dest);
 677         ra_modified(as, dest);
 678         emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSD : XO_MOVSS,
 679                   dest, RID_ESP, ofs);
 680       }
 681       if ((ci->flags & CCI_CASTU64)) {
 682         emit_movtomro(as, RID_RETLO, RID_ESP, ofs);
 683         emit_movtomro(as, RID_RETHI, RID_ESP, ofs+4);
 684       } else {
 685         emit_rmro(as, irt_isnum(ir->t) ? XO_FSTPq : XO_FSTPd,
 686                   irt_isnum(ir->t) ? XOg_FSTPq : XOg_FSTPd, RID_ESP, ofs);
 687       }
 688 #endif
 689 #if LJ_32
 690     } else if (hiop) {
 691       ra_destpair(as, ir);
 692 #endif
 693     } else {
 694       lua_assert(!irt_ispri(ir->t));
 695       ra_destreg(as, ir, RID_RET);
 696     }
 697   } else if (LJ_32 && irt_isfp(ir->t) && !(ci->flags & CCI_CASTU64)) {
 698     emit_x87op(as, XI_FPOP);  /* Pop unused result from x87 st0. */
 699   }
 700 }
 701 
 702 /* Return a constant function pointer or NULL for indirect calls. */
 703 static void *asm_callx_func(ASMState *as, IRIns *irf, IRRef func)
 704 {
 705 #if LJ_32
 706   UNUSED(as);
 707   if (irref_isk(func))
 708     return (void *)irf->i;
 709 #else
 710   if (irref_isk(func)) {
 711     MCode *p;
 712     if (irf->o == IR_KINT64)
 713       p = (MCode *)(void *)ir_k64(irf)->u64;
 714     else
 715       p = (MCode *)(void *)(uintptr_t)(uint32_t)irf->i;
 716     if (p - as->mcp == (int32_t)(p - as->mcp))
 717       return p;  /* Call target is still in +-2GB range. */
 718     /* Avoid the indirect case of emit_call(). Try to hoist func addr. */
 719   }
 720 #endif
 721   return NULL;
 722 }
 723 
 724 static void asm_callx(ASMState *as, IRIns *ir)
 725 {
 726   IRRef args[CCI_NARGS_MAX*2];
 727   CCallInfo ci;
 728   IRRef func;
 729   IRIns *irf;
 730   int32_t spadj = 0;
 731   ci.flags = asm_callx_flags(as, ir);
 732   asm_collectargs(as, ir, &ci, args);
 733   asm_setupresult(as, ir, &ci);
 734 #if LJ_32
 735   /* Have to readjust stack after non-cdecl calls due to callee cleanup. */
 736   if ((ci.flags & CCI_CC_MASK) != CCI_CC_CDECL)
 737     spadj = 4 * asm_count_call_slots(as, &ci, args);
 738 #endif
 739   func = ir->op2; irf = IR(func);
 740   if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); }
 741   ci.func = (ASMFunction)asm_callx_func(as, irf, func);
 742   if (!(void *)ci.func) {
 743     /* Use a (hoistable) non-scratch register for indirect calls. */
 744     RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
 745     Reg r = ra_alloc1(as, func, allow);
 746     if (LJ_32) emit_spsub(as, spadj);  /* Above code may cause restores! */
 747     emit_rr(as, XO_GROUP5, XOg_CALL, r);
 748   } else if (LJ_32) {
 749     emit_spsub(as, spadj);
 750   }
 751   asm_gencall(as, &ci, args);
 752 }
 753 
 754 /* -- Returns ------------------------------------------------------------- */
 755 
 756 /* Return to lower frame. Guard that it goes to the right spot. */
 757 static void asm_retf(ASMState *as, IRIns *ir)
 758 {
 759   Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
 760 #if LJ_FR2
 761   Reg rpc = ra_scratch(as, rset_exclude(RSET_GPR, base));
 762 #endif
 763   void *pc = ir_kptr(IR(ir->op2));
 764   int32_t delta = 1+LJ_FR2+bc_a(*((const BCIns *)pc - 1));
 765   as->topslot -= (BCReg)delta;
 766   if ((int32_t)as->topslot < 0) as->topslot = 0;
 767   irt_setmark(IR(REF_BASE)->t);  /* Children must not coalesce with BASE reg. */
 768   emit_setgl(as, base, jit_base);
 769   emit_addptr(as, base, -8*delta);
 770   asm_guardcc(as, CC_NE);
 771 #if LJ_FR2
 772   emit_rmro(as, XO_CMP, rpc|REX_GC64, base, -8);
 773   emit_loadu64(as, rpc, u64ptr(pc));
 774 #else
 775   emit_gmroi(as, XG_ARITHi(XOg_CMP), base, -4, ptr2addr(pc));
 776 #endif
 777 }
 778 
 779 /* -- Type conversions ---------------------------------------------------- */
 780 
 781 static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
 782 {
 783   Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
 784   Reg dest = ra_dest(as, ir, RSET_GPR);
 785   asm_guardcc(as, CC_P);
 786   asm_guardcc(as, CC_NE);
 787   emit_rr(as, XO_UCOMISD, left, tmp);
 788   emit_rr(as, XO_CVTSI2SD, tmp, dest);
 789   emit_rr(as, XO_XORPS, tmp, tmp);  /* Avoid partial register stall. */
 790   emit_rr(as, XO_CVTTSD2SI, dest, left);
 791   /* Can't fuse since left is needed twice. */
 792 }
 793 
 794 static void asm_tobit(ASMState *as, IRIns *ir)
 795 {
 796   Reg dest = ra_dest(as, ir, RSET_GPR);
 797   Reg tmp = ra_noreg(IR(ir->op1)->r) ?
 798               ra_alloc1(as, ir->op1, RSET_FPR) :
 799               ra_scratch(as, RSET_FPR);
 800   Reg right;
 801   emit_rr(as, XO_MOVDto, tmp, dest);
 802   right = asm_fuseload(as, ir->op2, rset_exclude(RSET_FPR, tmp));
 803   emit_mrm(as, XO_ADDSD, tmp, right);
 804   ra_left(as, tmp, ir->op1);
 805 }
 806 
 807 static void asm_conv(ASMState *as, IRIns *ir)
 808 {
 809   IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
 810   int st64 = (st == IRT_I64 || st == IRT_U64 || (LJ_64 && st == IRT_P64));
 811   int stfp = (st == IRT_NUM || st == IRT_FLOAT);
 812   IRRef lref = ir->op1;
 813   lua_assert(irt_type(ir->t) != st);
 814   lua_assert(!(LJ_32 && (irt_isint64(ir->t) || st64)));  /* Handled by SPLIT. */
 815   if (irt_isfp(ir->t)) {
 816     Reg dest = ra_dest(as, ir, RSET_FPR);
 817     if (stfp) {  /* FP to FP conversion. */
 818       Reg left = asm_fuseload(as, lref, RSET_FPR);
 819       emit_mrm(as, st == IRT_NUM ? XO_CVTSD2SS : XO_CVTSS2SD, dest, left);
 820       if (left == dest) return;  /* Avoid the XO_XORPS. */
 821     } else if (LJ_32 && st == IRT_U32) {  /* U32 to FP conversion on x86. */
 822       /* number = (2^52+2^51 .. u32) - (2^52+2^51) */
 823       cTValue *k = &as->J->k64[LJ_K64_TOBIT];
 824       Reg bias = ra_scratch(as, rset_exclude(RSET_FPR, dest));
 825       if (irt_isfloat(ir->t))
 826         emit_rr(as, XO_CVTSD2SS, dest, dest);
 827       emit_rr(as, XO_SUBSD, dest, bias);  /* Subtract 2^52+2^51 bias. */
 828       emit_rr(as, XO_XORPS, dest, bias);  /* Merge bias and integer. */
 829       emit_rma(as, XO_MOVSD, bias, k);
 830       emit_mrm(as, XO_MOVD, dest, asm_fuseload(as, lref, RSET_GPR));
 831       return;
 832     } else {  /* Integer to FP conversion. */
 833       Reg left = (LJ_64 && (st == IRT_U32 || st == IRT_U64)) ?
 834                  ra_alloc1(as, lref, RSET_GPR) :
 835                  asm_fuseloadm(as, lref, RSET_GPR, st64);
 836       if (LJ_64 && st == IRT_U64) {
 837         MCLabel l_end = emit_label(as);
 838         cTValue *k = &as->J->k64[LJ_K64_2P64];
 839         emit_rma(as, XO_ADDSD, dest, k);  /* Add 2^64 to compensate. */
 840         emit_sjcc(as, CC_NS, l_end);
 841         emit_rr(as, XO_TEST, left|REX_64, left);  /* Check if u64 >= 2^63. */
 842       }
 843       emit_mrm(as, irt_isnum(ir->t) ? XO_CVTSI2SD : XO_CVTSI2SS,
 844                dest|((LJ_64 && (st64 || st == IRT_U32)) ? REX_64 : 0), left);
 845     }
 846     emit_rr(as, XO_XORPS, dest, dest);  /* Avoid partial register stall. */
 847   } else if (stfp) {  /* FP to integer conversion. */
 848     if (irt_isguard(ir->t)) {
 849       /* Checked conversions are only supported from number to int. */
 850       lua_assert(irt_isint(ir->t) && st == IRT_NUM);
 851       asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
 852     } else {
 853       Reg dest = ra_dest(as, ir, RSET_GPR);
 854       x86Op op = st == IRT_NUM ? XO_CVTTSD2SI : XO_CVTTSS2SI;
 855       if (LJ_64 ? irt_isu64(ir->t) : irt_isu32(ir->t)) {
 856         /* LJ_64: For inputs >= 2^63 add -2^64, convert again. */
 857         /* LJ_32: For inputs >= 2^31 add -2^31, convert again and add 2^31. */
 858         Reg tmp = ra_noreg(IR(lref)->r) ? ra_alloc1(as, lref, RSET_FPR) :
 859                                           ra_scratch(as, RSET_FPR);
 860         MCLabel l_end = emit_label(as);
 861         if (LJ_32)
 862           emit_gri(as, XG_ARITHi(XOg_ADD), dest, (int32_t)0x80000000);
 863         emit_rr(as, op, dest|REX_64, tmp);
 864         if (st == IRT_NUM)
 865           emit_rma(as, XO_ADDSD, tmp, &as->J->k64[LJ_K64_M2P64_31]);
 866         else
 867           emit_rma(as, XO_ADDSS, tmp, &as->J->k32[LJ_K32_M2P64_31]);
 868         emit_sjcc(as, CC_NS, l_end);
 869         emit_rr(as, XO_TEST, dest|REX_64, dest);  /* Check if dest negative. */
 870         emit_rr(as, op, dest|REX_64, tmp);
 871         ra_left(as, tmp, lref);
 872       } else {
 873         if (LJ_64 && irt_isu32(ir->t))
 874           emit_rr(as, XO_MOV, dest, dest);  /* Zero hiword. */
 875         emit_mrm(as, op,
 876                  dest|((LJ_64 &&
 877                         (irt_is64(ir->t) || irt_isu32(ir->t))) ? REX_64 : 0),
 878                  asm_fuseload(as, lref, RSET_FPR));
 879       }
 880     }
 881   } else if (st >= IRT_I8 && st <= IRT_U16) {  /* Extend to 32 bit integer. */
 882     Reg left, dest = ra_dest(as, ir, RSET_GPR);
 883     RegSet allow = RSET_GPR;
 884     x86Op op;
 885     lua_assert(irt_isint(ir->t) || irt_isu32(ir->t));
 886     if (st == IRT_I8) {
 887       op = XO_MOVSXb; allow = RSET_GPR8; dest |= FORCE_REX;
 888     } else if (st == IRT_U8) {
 889       op = XO_MOVZXb; allow = RSET_GPR8; dest |= FORCE_REX;
 890     } else if (st == IRT_I16) {
 891       op = XO_MOVSXw;
 892     } else {
 893       op = XO_MOVZXw;
 894     }
 895     left = asm_fuseload(as, lref, allow);
 896     /* Add extra MOV if source is already in wrong register. */
 897     if (!LJ_64 && left != RID_MRM && !rset_test(allow, left)) {
 898       Reg tmp = ra_scratch(as, allow);
 899       emit_rr(as, op, dest, tmp);
 900       emit_rr(as, XO_MOV, tmp, left);
 901     } else {
 902       emit_mrm(as, op, dest, left);
 903     }
 904   } else {  /* 32/64 bit integer conversions. */
 905     if (LJ_32) {  /* Only need to handle 32/32 bit no-op (cast) on x86. */
 906       Reg dest = ra_dest(as, ir, RSET_GPR);
 907       ra_left(as, dest, lref);  /* Do nothing, but may need to move regs. */
 908     } else if (irt_is64(ir->t)) {
 909       Reg dest = ra_dest(as, ir, RSET_GPR);
 910       if (st64 || !(ir->op2 & IRCONV_SEXT)) {
 911         /* 64/64 bit no-op (cast) or 32 to 64 bit zero extension. */
 912         ra_left(as, dest, lref);  /* Do nothing, but may need to move regs. */
 913       } else {  /* 32 to 64 bit sign extension. */
 914         Reg left = asm_fuseload(as, lref, RSET_GPR);
 915         emit_mrm(as, XO_MOVSXd, dest|REX_64, left);
 916       }
 917     } else {
 918       Reg dest = ra_dest(as, ir, RSET_GPR);
 919       if (st64) {
 920         Reg left = asm_fuseload(as, lref, RSET_GPR);
 921         /* This is either a 32 bit reg/reg mov which zeroes the hiword
 922         ** or a load of the loword from a 64 bit address.
 923         */
 924         emit_mrm(as, XO_MOV, dest, left);
 925       } else {  /* 32/32 bit no-op (cast). */
 926         ra_left(as, dest, lref);  /* Do nothing, but may need to move regs. */
 927       }
 928     }
 929   }
 930 }
 931 
 932 #if LJ_32 && LJ_HASFFI
 933 /* No SSE conversions to/from 64 bit on x86, so resort to ugly x87 code. */
 934 
 935 /* 64 bit integer to FP conversion in 32 bit mode. */
 936 static void asm_conv_fp_int64(ASMState *as, IRIns *ir)
 937 {
 938   Reg hi = ra_alloc1(as, ir->op1, RSET_GPR);
 939   Reg lo = ra_alloc1(as, (ir-1)->op1, rset_exclude(RSET_GPR, hi));
 940   int32_t ofs = sps_scale(ir->s);  /* Use spill slot or temp slots. */
 941   Reg dest = ir->r;
 942   if (ra_hasreg(dest)) {
 943     ra_free(as, dest);
 944     ra_modified(as, dest);
 945     emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSD : XO_MOVSS, dest, RID_ESP, ofs);
 946   }
 947   emit_rmro(as, irt_isnum(ir->t) ? XO_FSTPq : XO_FSTPd,
 948             irt_isnum(ir->t) ? XOg_FSTPq : XOg_FSTPd, RID_ESP, ofs);
 949   if (((ir-1)->op2 & IRCONV_SRCMASK) == IRT_U64) {
 950     /* For inputs in [2^63,2^64-1] add 2^64 to compensate. */
 951     MCLabel l_end = emit_label(as);
 952     emit_rma(as, XO_FADDq, XOg_FADDq, &as->J->k64[LJ_K64_2P64]);
 953     emit_sjcc(as, CC_NS, l_end);
 954     emit_rr(as, XO_TEST, hi, hi);  /* Check if u64 >= 2^63. */
 955   } else {
 956     lua_assert(((ir-1)->op2 & IRCONV_SRCMASK) == IRT_I64);
 957   }
 958   emit_rmro(as, XO_FILDq, XOg_FILDq, RID_ESP, 0);
 959   /* NYI: Avoid narrow-to-wide store-to-load forwarding stall. */
 960   emit_rmro(as, XO_MOVto, hi, RID_ESP, 4);
 961   emit_rmro(as, XO_MOVto, lo, RID_ESP, 0);
 962 }
 963 
 964 /* FP to 64 bit integer conversion in 32 bit mode. */
 965 static void asm_conv_int64_fp(ASMState *as, IRIns *ir)
 966 {
 967   IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
 968   IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
 969   Reg lo, hi;
 970   lua_assert(st == IRT_NUM || st == IRT_FLOAT);
 971   lua_assert(dt == IRT_I64 || dt == IRT_U64);
 972   hi = ra_dest(as, ir, RSET_GPR);
 973   lo = ra_dest(as, ir-1, rset_exclude(RSET_GPR, hi));
 974   if (ra_used(ir-1)) emit_rmro(as, XO_MOV, lo, RID_ESP, 0);
 975   /* NYI: Avoid wide-to-narrow store-to-load forwarding stall. */
 976   if (!(as->flags & JIT_F_SSE3)) {  /* Set FPU rounding mode to default. */
 977     emit_rmro(as, XO_FLDCW, XOg_FLDCW, RID_ESP, 4);
 978     emit_rmro(as, XO_MOVto, lo, RID_ESP, 4);
 979     emit_gri(as, XG_ARITHi(XOg_AND), lo, 0xf3ff);
 980   }
 981   if (dt == IRT_U64) {
 982     /* For inputs in [2^63,2^64-1] add -2^64 and convert again. */
 983     MCLabel l_pop, l_end = emit_label(as);
 984     emit_x87op(as, XI_FPOP);
 985     l_pop = emit_label(as);
 986     emit_sjmp(as, l_end);
 987     emit_rmro(as, XO_MOV, hi, RID_ESP, 4);
 988     if ((as->flags & JIT_F_SSE3))
 989       emit_rmro(as, XO_FISTTPq, XOg_FISTTPq, RID_ESP, 0);
 990     else
 991       emit_rmro(as, XO_FISTPq, XOg_FISTPq, RID_ESP, 0);
 992     emit_rma(as, XO_FADDq, XOg_FADDq, &as->J->k64[LJ_K64_M2P64]);
 993     emit_sjcc(as, CC_NS, l_pop);
 994     emit_rr(as, XO_TEST, hi, hi);  /* Check if out-of-range (2^63). */
 995   }
 996   emit_rmro(as, XO_MOV, hi, RID_ESP, 4);
 997   if ((as->flags & JIT_F_SSE3)) {  /* Truncation is easy with SSE3. */
 998     emit_rmro(as, XO_FISTTPq, XOg_FISTTPq, RID_ESP, 0);
 999   } else {  /* Otherwise set FPU rounding mode to truncate before the store. */
1000     emit_rmro(as, XO_FISTPq, XOg_FISTPq, RID_ESP, 0);
1001     emit_rmro(as, XO_FLDCW, XOg_FLDCW, RID_ESP, 0);
1002     emit_rmro(as, XO_MOVtow, lo, RID_ESP, 0);
1003     emit_rmro(as, XO_ARITHw(XOg_OR), lo, RID_ESP, 0);
1004     emit_loadi(as, lo, 0xc00);
1005     emit_rmro(as, XO_FNSTCW, XOg_FNSTCW, RID_ESP, 0);
1006   }
1007   if (dt == IRT_U64)
1008     emit_x87op(as, XI_FDUP);
1009   emit_mrm(as, st == IRT_NUM ? XO_FLDq : XO_FLDd,
1010            st == IRT_NUM ? XOg_FLDq: XOg_FLDd,
1011            asm_fuseload(as, ir->op1, RSET_EMPTY));
1012 }
1013 
1014 static void asm_conv64(ASMState *as, IRIns *ir)
1015 {
1016   if (irt_isfp(ir->t))
1017     asm_conv_fp_int64(as, ir);
1018   else
1019     asm_conv_int64_fp(as, ir);
1020 }
1021 #endif
1022 
1023 static void asm_strto(ASMState *as, IRIns *ir)
1024 {
1025   /* Force a spill slot for the destination register (if any). */
1026   const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
1027   IRRef args[2];
1028   RegSet drop = RSET_SCRATCH;
1029   if ((drop & RSET_FPR) != RSET_FPR && ra_hasreg(ir->r))
1030     rset_set(drop, ir->r);  /* WIN64 doesn't spill all FPRs. */
1031   ra_evictset(as, drop);
1032   asm_guardcc(as, CC_E);
1033   emit_rr(as, XO_TEST, RID_RET, RID_RET);  /* Test return status. */
1034   args[0] = ir->op1;      /* GCstr *str */
1035   args[1] = ASMREF_TMP1;  /* TValue *n  */
1036   asm_gencall(as, ci, args);
1037   /* Store the result to the spill slot or temp slots. */
1038   emit_rmro(as, XO_LEA, ra_releasetmp(as, ASMREF_TMP1)|REX_64,
1039             RID_ESP, sps_scale(ir->s));
1040 }
1041 
1042 /* -- Memory references --------------------------------------------------- */
1043 
1044 /* Get pointer to TValue. */
1045 static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
1046 {
1047   IRIns *ir = IR(ref);
1048   if (irt_isnum(ir->t)) {
1049     /* For numbers use the constant itself or a spill slot as a TValue. */
1050     if (irref_isk(ref))
1051       emit_loada(as, dest, ir_knum(ir));
1052     else
1053       emit_rmro(as, XO_LEA, dest|REX_64, RID_ESP, ra_spill(as, ir));
1054   } else {
1055     /* Otherwise use g->tmptv to hold the TValue. */
1056 #if LJ_GC64
1057     if (irref_isk(ref)) {
1058       TValue k;
1059       lj_ir_kvalue(as->J->L, &k, ir);
1060       emit_movmroi(as, dest, 4, k.u32.hi);
1061       emit_movmroi(as, dest, 0, k.u32.lo);
1062     } else {
1063       /* TODO: 64 bit store + 32 bit load-modify-store is suboptimal. */
1064       Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, dest));
1065       if (irt_is64(ir->t)) {
1066         emit_u32(as, irt_toitype(ir->t) << 15);
1067         emit_rmro(as, XO_ARITHi, XOg_OR, dest, 4);
1068       } else {
1069         /* Currently, no caller passes integers that might end up here. */
1070         emit_movmroi(as, dest, 4, (irt_toitype(ir->t) << 15));
1071       }
1072       emit_movtomro(as, REX_64IR(ir, src), dest, 0);
1073     }
1074 #else
1075     if (!irref_isk(ref)) {
1076       Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, dest));
1077       emit_movtomro(as, REX_64IR(ir, src), dest, 0);
1078     } else if (!irt_ispri(ir->t)) {
1079       emit_movmroi(as, dest, 0, ir->i);
1080     }
1081     if (!(LJ_64 && irt_islightud(ir->t)))
1082       emit_movmroi(as, dest, 4, irt_toitype(ir->t));
1083 #endif
1084     emit_loada(as, dest, &J2G(as->J)->tmptv);
1085   }
1086 }
1087 
1088 static void asm_aref(ASMState *as, IRIns *ir)
1089 {
1090   Reg dest = ra_dest(as, ir, RSET_GPR);
1091   asm_fusearef(as, ir, RSET_GPR);
1092   if (!(as->mrm.idx == RID_NONE && as->mrm.ofs == 0))
1093     emit_mrm(as, XO_LEA, dest|REX_GC64, RID_MRM);
1094   else if (as->mrm.base != dest)
1095     emit_rr(as, XO_MOV, dest|REX_GC64, as->mrm.base);
1096 }
1097 
1098 /* Inlined hash lookup. Specialized for key type and for const keys.
1099 ** The equivalent C code is:
1100 **   Node *n = hashkey(t, key);
1101 **   do {
1102 **     if (lj_obj_equal(&n->key, key)) return &n->val;
1103 **   } while ((n = nextnode(n)));
1104 **   return niltv(L);
1105 */
1106 static void asm_href(ASMState *as, IRIns *ir, IROp merge)
1107 {
1108   RegSet allow = RSET_GPR;
1109   int destused = ra_used(ir);
1110   Reg dest = ra_dest(as, ir, allow);
1111   Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest));
1112   Reg key = RID_NONE, tmp = RID_NONE;
1113   IRIns *irkey = IR(ir->op2);
1114   int isk = irref_isk(ir->op2);
1115   IRType1 kt = irkey->t;
1116   uint32_t khash;
1117   MCLabel l_end, l_loop, l_next;
1118 
1119   if (!isk) {
1120     rset_clear(allow, tab);
1121     key = ra_alloc1(as, ir->op2, irt_isnum(kt) ? RSET_FPR : allow);
1122     if (LJ_GC64 || !irt_isstr(kt))
1123       tmp = ra_scratch(as, rset_exclude(allow, key));
1124   }
1125 
1126   /* Key not found in chain: jump to exit (if merged) or load niltv. */
1127   l_end = emit_label(as);
1128   if (merge == IR_NE)
1129     asm_guardcc(as, CC_E);  /* XI_JMP is not found by lj_asm_patchexit. */
1130   else if (destused)
1131     emit_loada(as, dest, niltvg(J2G(as->J)));
1132 
1133   /* Follow hash chain until the end. */
1134   l_loop = emit_sjcc_label(as, CC_NZ);
1135   emit_rr(as, XO_TEST, dest|REX_GC64, dest);
1136   emit_rmro(as, XO_MOV, dest|REX_GC64, dest, offsetof(Node, next));
1137   l_next = emit_label(as);
1138 
1139   /* Type and value comparison. */
1140   if (merge == IR_EQ)
1141     asm_guardcc(as, CC_E);
1142   else
1143     emit_sjcc(as, CC_E, l_end);
1144   if (irt_isnum(kt)) {
1145     if (isk) {
1146       /* Assumes -0.0 is already canonicalized to +0.0. */
1147       emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.u32.lo),
1148                  (int32_t)ir_knum(irkey)->u32.lo);
1149       emit_sjcc(as, CC_NE, l_next);
1150       emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.u32.hi),
1151                  (int32_t)ir_knum(irkey)->u32.hi);
1152     } else {
1153       emit_sjcc(as, CC_P, l_next);
1154       emit_rmro(as, XO_UCOMISD, key, dest, offsetof(Node, key.n));
1155       emit_sjcc(as, CC_AE, l_next);
1156       /* The type check avoids NaN penalties and complaints from Valgrind. */
1157 #if LJ_64 && !LJ_GC64
1158       emit_u32(as, LJ_TISNUM);
1159       emit_rmro(as, XO_ARITHi, XOg_CMP, dest, offsetof(Node, key.it));
1160 #else
1161       emit_i8(as, LJ_TISNUM);
1162       emit_rmro(as, XO_ARITHi8, XOg_CMP, dest, offsetof(Node, key.it));
1163 #endif
1164     }
1165 #if LJ_64 && !LJ_GC64
1166   } else if (irt_islightud(kt)) {
1167     emit_rmro(as, XO_CMP, key|REX_64, dest, offsetof(Node, key.u64));
1168 #endif
1169 #if LJ_GC64
1170   } else if (irt_isaddr(kt)) {
1171     if (isk) {
1172       TValue k;
1173       k.u64 = ((uint64_t)irt_toitype(irkey->t) << 47) | irkey[1].tv.u64;
1174       emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.u32.lo),
1175                  k.u32.lo);
1176       emit_sjcc(as, CC_NE, l_next);
1177       emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.u32.hi),
1178                  k.u32.hi);
1179     } else {
1180       emit_rmro(as, XO_CMP, tmp|REX_64, dest, offsetof(Node, key.u64));
1181     }
1182   } else {
1183     lua_assert(irt_ispri(kt) && !irt_isnil(kt));
1184     emit_u32(as, (irt_toitype(kt)<<15)|0x7fff);
1185     emit_rmro(as, XO_ARITHi, XOg_CMP, dest, offsetof(Node, key.it));
1186 #else
1187   } else {
1188     if (!irt_ispri(kt)) {
1189       lua_assert(irt_isaddr(kt));
1190       if (isk)
1191         emit_gmroi(as, XG_ARITHi(XOg_CMP), dest, offsetof(Node, key.gcr),
1192                    ptr2addr(ir_kgc(irkey)));
1193       else
1194         emit_rmro(as, XO_CMP, key, dest, offsetof(Node, key.gcr));
1195       emit_sjcc(as, CC_NE, l_next);
1196     }
1197     lua_assert(!irt_isnil(kt));
1198     emit_i8(as, irt_toitype(kt));
1199     emit_rmro(as, XO_ARITHi8, XOg_CMP, dest, offsetof(Node, key.it));
1200 #endif
1201   }
1202   emit_sfixup(as, l_loop);
1203   checkmclim(as);
1204 #if LJ_GC64
1205   if (!isk && irt_isaddr(kt)) {
1206     emit_rr(as, XO_OR, tmp|REX_64, key);
1207     emit_loadu64(as, tmp, (uint64_t)irt_toitype(kt) << 47);
1208   }
1209 #endif
1210 
1211   /* Load main position relative to tab->node into dest. */
1212   khash = isk ? ir_khash(irkey) : 1;
1213   if (khash == 0) {
1214     emit_rmro(as, XO_MOV, dest|REX_GC64, tab, offsetof(GCtab, node));
1215   } else {
1216     emit_rmro(as, XO_ARITH(XOg_ADD), dest|REX_GC64, tab, offsetof(GCtab,node));
1217     if ((as->flags & JIT_F_PREFER_IMUL)) {
1218       emit_i8(as, sizeof(Node));
1219       emit_rr(as, XO_IMULi8, dest, dest);
1220     } else {
1221       emit_shifti(as, XOg_SHL, dest, 3);
1222       emit_rmrxo(as, XO_LEA, dest, dest, dest, XM_SCALE2, 0);
1223     }
1224     if (isk) {
1225       emit_gri(as, XG_ARITHi(XOg_AND), dest, (int32_t)khash);
1226       emit_rmro(as, XO_MOV, dest, tab, offsetof(GCtab, hmask));
1227     } else if (irt_isstr(kt)) {
1228       emit_rmro(as, XO_ARITH(XOg_AND), dest, key, offsetof(GCstr, hash));
1229       emit_rmro(as, XO_MOV, dest, tab, offsetof(GCtab, hmask));
1230     } else {  /* Must match with hashrot() in lj_tab.c. */
1231       emit_rmro(as, XO_ARITH(XOg_AND), dest, tab, offsetof(GCtab, hmask));
1232       emit_rr(as, XO_ARITH(XOg_SUB), dest, tmp);
1233       emit_shifti(as, XOg_ROL, tmp, HASH_ROT3);
1234       emit_rr(as, XO_ARITH(XOg_XOR), dest, tmp);
1235       emit_shifti(as, XOg_ROL, dest, HASH_ROT2);
1236       emit_rr(as, XO_ARITH(XOg_SUB), tmp, dest);
1237       emit_shifti(as, XOg_ROL, dest, HASH_ROT1);
1238       emit_rr(as, XO_ARITH(XOg_XOR), tmp, dest);
1239       if (irt_isnum(kt)) {
1240         emit_rr(as, XO_ARITH(XOg_ADD), dest, dest);
1241 #if LJ_64
1242         emit_shifti(as, XOg_SHR|REX_64, dest, 32);
1243         emit_rr(as, XO_MOV, tmp, dest);
1244         emit_rr(as, XO_MOVDto, key|REX_64, dest);
1245 #else
1246         emit_rmro(as, XO_MOV, dest, RID_ESP, ra_spill(as, irkey)+4);
1247         emit_rr(as, XO_MOVDto, key, tmp);
1248 #endif
1249       } else {
1250         emit_rr(as, XO_MOV, tmp, key);
1251 #if LJ_GC64
1252         checkmclim(as);
1253         emit_gri(as, XG_ARITHi(XOg_XOR), dest, irt_toitype(kt) << 15);
1254         if ((as->flags & JIT_F_BMI2)) {
1255           emit_i8(as, 32);
1256           emit_mrm(as, XV_RORX|VEX_64, dest, key);
1257         } else {
1258           emit_shifti(as, XOg_SHR|REX_64, dest, 32);
1259           emit_rr(as, XO_MOV, dest|REX_64, key|REX_64);
1260         }
1261 #else
1262         emit_rmro(as, XO_LEA, dest, key, HASH_BIAS);
1263 #endif
1264       }
1265     }
1266   }
1267 }
1268 
1269 static void asm_hrefk(ASMState *as, IRIns *ir)
1270 {
1271   IRIns *kslot = IR(ir->op2);
1272   IRIns *irkey = IR(kslot->op1);
1273   int32_t ofs = (int32_t)(kslot->op2 * sizeof(Node));
1274   Reg dest = ra_used(ir) ? ra_dest(as, ir, RSET_GPR) : RID_NONE;
1275   Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
1276 #if !LJ_64
1277   MCLabel l_exit;
1278 #endif
1279   lua_assert(ofs % sizeof(Node) == 0);
1280   if (ra_hasreg(dest)) {
1281     if (ofs != 0) {
1282       if (dest == node && !(as->flags & JIT_F_LEA_AGU))
1283         emit_gri(as, XG_ARITHi(XOg_ADD), dest|REX_GC64, ofs);
1284       else
1285         emit_rmro(as, XO_LEA, dest|REX_GC64, node, ofs);
1286     } else if (dest != node) {
1287       emit_rr(as, XO_MOV, dest|REX_GC64, node);
1288     }
1289   }
1290   asm_guardcc(as, CC_NE);
1291 #if LJ_64
1292   if (!irt_ispri(irkey->t)) {
1293     Reg key = ra_scratch(as, rset_exclude(RSET_GPR, node));
1294     emit_rmro(as, XO_CMP, key|REX_64, node,
1295                ofs + (int32_t)offsetof(Node, key.u64));
1296     lua_assert(irt_isnum(irkey->t) || irt_isgcv(irkey->t));
1297     /* Assumes -0.0 is already canonicalized to +0.0. */
1298     emit_loadu64(as, key, irt_isnum(irkey->t) ? ir_knum(irkey)->u64 :
1299 #if LJ_GC64
1300                           ((uint64_t)irt_toitype(irkey->t) << 47) |
1301                           (uint64_t)ir_kgc(irkey));
1302 #else
1303                           ((uint64_t)irt_toitype(irkey->t) << 32) |
1304                           (uint64_t)(uint32_t)ptr2addr(ir_kgc(irkey)));
1305 #endif
1306   } else {
1307     lua_assert(!irt_isnil(irkey->t));
1308 #if LJ_GC64
1309     emit_i32(as, (irt_toitype(irkey->t)<<15)|0x7fff);
1310     emit_rmro(as, XO_ARITHi, XOg_CMP, node,
1311               ofs + (int32_t)offsetof(Node, key.it));
1312 #else
1313     emit_i8(as, irt_toitype(irkey->t));
1314     emit_rmro(as, XO_ARITHi8, XOg_CMP, node,
1315               ofs + (int32_t)offsetof(Node, key.it));
1316 #endif
1317   }
1318 #else
1319   l_exit = emit_label(as);
1320   if (irt_isnum(irkey->t)) {
1321     /* Assumes -0.0 is already canonicalized to +0.0. */
1322     emit_gmroi(as, XG_ARITHi(XOg_CMP), node,
1323                ofs + (int32_t)offsetof(Node, key.u32.lo),
1324                (int32_t)ir_knum(irkey)->u32.lo);
1325     emit_sjcc(as, CC_NE, l_exit);
1326     emit_gmroi(as, XG_ARITHi(XOg_CMP), node,
1327                ofs + (int32_t)offsetof(Node, key.u32.hi),
1328                (int32_t)ir_knum(irkey)->u32.hi);
1329   } else {
1330     if (!irt_ispri(irkey->t)) {
1331       lua_assert(irt_isgcv(irkey->t));
1332       emit_gmroi(as, XG_ARITHi(XOg_CMP), node,
1333                  ofs + (int32_t)offsetof(Node, key.gcr),
1334                  ptr2addr(ir_kgc(irkey)));
1335       emit_sjcc(as, CC_NE, l_exit);
1336     }
1337     lua_assert(!irt_isnil(irkey->t));
1338     emit_i8(as, irt_toitype(irkey->t));
1339     emit_rmro(as, XO_ARITHi8, XOg_CMP, node,
1340               ofs + (int32_t)offsetof(Node, key.it));
1341   }
1342 #endif
1343 }
1344 
1345 static void asm_uref(ASMState *as, IRIns *ir)
1346 {
1347   Reg dest = ra_dest(as, ir, RSET_GPR);
1348   if (irref_isk(ir->op1)) {
1349     GCfunc *fn = ir_kfunc(IR(ir->op1));
1350     MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
1351     emit_rma(as, XO_MOV, dest|REX_GC64, v);
1352   } else {
1353     Reg uv = ra_scratch(as, RSET_GPR);
1354     Reg func = ra_alloc1(as, ir->op1, RSET_GPR);
1355     if (ir->o == IR_UREFC) {
1356       emit_rmro(as, XO_LEA, dest|REX_GC64, uv, offsetof(GCupval, tv));
1357       asm_guardcc(as, CC_NE);
1358       emit_i8(as, 1);
1359       emit_rmro(as, XO_ARITHib, XOg_CMP, uv, offsetof(GCupval, closed));
1360     } else {
1361       emit_rmro(as, XO_MOV, dest|REX_GC64, uv, offsetof(GCupval, v));
1362     }
1363     emit_rmro(as, XO_MOV, uv|REX_GC64, func,
1364               (int32_t)offsetof(GCfuncL, uvptr) +
1365               (int32_t)sizeof(MRef) * (int32_t)(ir->op2 >> 8));
1366   }
1367 }
1368 
1369 static void asm_fref(ASMState *as, IRIns *ir)
1370 {
1371   Reg dest = ra_dest(as, ir, RSET_GPR);
1372   asm_fusefref(as, ir, RSET_GPR);
1373   emit_mrm(as, XO_LEA, dest, RID_MRM);
1374 }
1375 
1376 static void asm_strref(ASMState *as, IRIns *ir)
1377 {
1378   Reg dest = ra_dest(as, ir, RSET_GPR);
1379   asm_fusestrref(as, ir, RSET_GPR);
1380   if (as->mrm.base == RID_NONE)
1381     emit_loadi(as, dest, as->mrm.ofs);
1382   else if (as->mrm.base == dest && as->mrm.idx == RID_NONE)
1383     emit_gri(as, XG_ARITHi(XOg_ADD), dest|REX_GC64, as->mrm.ofs);
1384   else
1385     emit_mrm(as, XO_LEA, dest|REX_GC64, RID_MRM);
1386 }
1387 
1388 /* -- Loads and stores ---------------------------------------------------- */
1389 
1390 static void asm_fxload(ASMState *as, IRIns *ir)
1391 {
1392   Reg dest = ra_dest(as, ir, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
1393   x86Op xo;
1394   if (ir->o == IR_FLOAD)
1395     asm_fusefref(as, ir, RSET_GPR);
1396   else
1397     asm_fusexref(as, ir->op1, RSET_GPR);
1398   /* ir->op2 is ignored -- unaligned loads are ok on x86. */
1399   switch (irt_type(ir->t)) {
1400   case IRT_I8: xo = XO_MOVSXb; break;
1401   case IRT_U8: xo = XO_MOVZXb; break;
1402   case IRT_I16: xo = XO_MOVSXw; break;
1403   case IRT_U16: xo = XO_MOVZXw; break;
1404   case IRT_NUM: xo = XO_MOVSD; break;
1405   case IRT_FLOAT: xo = XO_MOVSS; break;
1406   default:
1407     if (LJ_64 && irt_is64(ir->t))
1408       dest |= REX_64;
1409     else
1410       lua_assert(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t));
1411     xo = XO_MOV;
1412     break;
1413   }
1414   emit_mrm(as, xo, dest, RID_MRM);
1415 }
1416 
1417 #define asm_fload(as, ir)       asm_fxload(as, ir)
1418 #define asm_xload(as, ir)       asm_fxload(as, ir)
1419 
1420 static void asm_fxstore(ASMState *as, IRIns *ir)
1421 {
1422   RegSet allow = RSET_GPR;
1423   Reg src = RID_NONE, osrc = RID_NONE;
1424   int32_t k = 0;
1425   if (ir->r == RID_SINK)
1426     return;
1427   /* The IRT_I16/IRT_U16 stores should never be simplified for constant
1428   ** values since mov word [mem], imm16 has a length-changing prefix.
1429   */
1430   if (irt_isi16(ir->t) || irt_isu16(ir->t) || irt_isfp(ir->t) ||
1431       !asm_isk32(as, ir->op2, &k)) {
1432     RegSet allow8 = irt_isfp(ir->t) ? RSET_FPR :
1433                     (irt_isi8(ir->t) || irt_isu8(ir->t)) ? RSET_GPR8 : RSET_GPR;
1434     src = osrc = ra_alloc1(as, ir->op2, allow8);
1435     if (!LJ_64 && !rset_test(allow8, src)) {  /* Already in wrong register. */
1436       rset_clear(allow, osrc);
1437       src = ra_scratch(as, allow8);
1438     }
1439     rset_clear(allow, src);
1440   }
1441   if (ir->o == IR_FSTORE) {
1442     asm_fusefref(as, IR(ir->op1), allow);
1443   } else {
1444     asm_fusexref(as, ir->op1, allow);
1445     if (LJ_32 && ir->o == IR_HIOP) as->mrm.ofs += 4;
1446   }
1447   if (ra_hasreg(src)) {
1448     x86Op xo;
1449     switch (irt_type(ir->t)) {
1450     case IRT_I8: case IRT_U8: xo = XO_MOVtob; src |= FORCE_REX; break;
1451     case IRT_I16: case IRT_U16: xo = XO_MOVtow; break;
1452     case IRT_NUM: xo = XO_MOVSDto; break;
1453     case IRT_FLOAT: xo = XO_MOVSSto; break;
1454 #if LJ_64 && !LJ_GC64
1455     case IRT_LIGHTUD: lua_assert(0);  /* NYI: mask 64 bit lightuserdata. */
1456 #endif
1457     default:
1458       if (LJ_64 && irt_is64(ir->t))
1459         src |= REX_64;
1460       else
1461         lua_assert(irt_isint(ir->t) || irt_isu32(ir->t) || irt_isaddr(ir->t));
1462       xo = XO_MOVto;
1463       break;
1464     }
1465     emit_mrm(as, xo, src, RID_MRM);
1466     if (!LJ_64 && src != osrc) {
1467       ra_noweak(as, osrc);
1468       emit_rr(as, XO_MOV, src, osrc);
1469     }
1470   } else {
1471     if (irt_isi8(ir->t) || irt_isu8(ir->t)) {
1472       emit_i8(as, k);
1473       emit_mrm(as, XO_MOVmib, 0, RID_MRM);
1474     } else {
1475       lua_assert(irt_is64(ir->t) || irt_isint(ir->t) || irt_isu32(ir->t) ||
1476                  irt_isaddr(ir->t));
1477       emit_i32(as, k);
1478       emit_mrm(as, XO_MOVmi, REX_64IR(ir, 0), RID_MRM);
1479     }
1480   }
1481 }
1482 
1483 #define asm_fstore(as, ir)      asm_fxstore(as, ir)
1484 #define asm_xstore(as, ir)      asm_fxstore(as, ir)
1485 
1486 #if LJ_64 && !LJ_GC64
1487 static Reg asm_load_lightud64(ASMState *as, IRIns *ir, int typecheck)
1488 {
1489   if (ra_used(ir) || typecheck) {
1490     Reg dest = ra_dest(as, ir, RSET_GPR);
1491     if (typecheck) {
1492       Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, dest));
1493       asm_guardcc(as, CC_NE);
1494       emit_i8(as, -2);
1495       emit_rr(as, XO_ARITHi8, XOg_CMP, tmp);
1496       emit_shifti(as, XOg_SAR|REX_64, tmp, 47);
1497       emit_rr(as, XO_MOV, tmp|REX_64, dest);
1498     }
1499     return dest;
1500   } else {
1501     return RID_NONE;
1502   }
1503 }
1504 #endif
1505 
1506 static void asm_ahuvload(ASMState *as, IRIns *ir)
1507 {
1508 #if LJ_GC64
1509   Reg tmp = RID_NONE;
1510 #endif
1511   lua_assert(irt_isnum(ir->t) || irt_ispri(ir->t) || irt_isaddr(ir->t) ||
1512              (LJ_DUALNUM && irt_isint(ir->t)));
1513 #if LJ_64 && !LJ_GC64
1514   if (irt_islightud(ir->t)) {
1515     Reg dest = asm_load_lightud64(as, ir, 1);
1516     if (ra_hasreg(dest)) {
1517       asm_fuseahuref(as, ir->op1, RSET_GPR);
1518       emit_mrm(as, XO_MOV, dest|REX_64, RID_MRM);
1519     }
1520     return;
1521   } else
1522 #endif
1523   if (ra_used(ir)) {
1524     RegSet allow = irt_isnum(ir->t) ? RSET_FPR : RSET_GPR;
1525     Reg dest = ra_dest(as, ir, allow);
1526     asm_fuseahuref(as, ir->op1, RSET_GPR);
1527 #if LJ_GC64
1528     if (irt_isaddr(ir->t)) {
1529       emit_shifti(as, XOg_SHR|REX_64, dest, 17);
1530       asm_guardcc(as, CC_NE);
1531       emit_i8(as, irt_toitype(ir->t));
1532       emit_rr(as, XO_ARITHi8, XOg_CMP, dest);
1533       emit_i8(as, XI_O16);
1534       if ((as->flags & JIT_F_BMI2)) {
1535         emit_i8(as, 47);
1536         emit_mrm(as, XV_RORX|VEX_64, dest, RID_MRM);
1537       } else {
1538         emit_shifti(as, XOg_ROR|REX_64, dest, 47);
1539         emit_mrm(as, XO_MOV, dest|REX_64, RID_MRM);
1540       }
1541       return;
1542     } else
1543 #endif
1544     emit_mrm(as, dest < RID_MAX_GPR ? XO_MOV : XO_MOVSD, dest, RID_MRM);
1545   } else {
1546     RegSet gpr = RSET_GPR;
1547 #if LJ_GC64
1548     if (irt_isaddr(ir->t)) {
1549       tmp = ra_scratch(as, RSET_GPR);
1550       gpr = rset_exclude(gpr, tmp);
1551     }
1552 #endif
1553     asm_fuseahuref(as, ir->op1, gpr);
1554   }
1555   /* Always do the type check, even if the load result is unused. */
1556   as->mrm.ofs += 4;
1557   asm_guardcc(as, irt_isnum(ir->t) ? CC_AE : CC_NE);
1558   if (LJ_64 && irt_type(ir->t) >= IRT_NUM) {
1559     lua_assert(irt_isinteger(ir->t) || irt_isnum(ir->t));
1560 #if LJ_GC64
1561     emit_u32(as, LJ_TISNUM << 15);
1562 #else
1563     emit_u32(as, LJ_TISNUM);
1564 #endif
1565     emit_mrm(as, XO_ARITHi, XOg_CMP, RID_MRM);
1566 #if LJ_GC64
1567   } else if (irt_isaddr(ir->t)) {
1568     as->mrm.ofs -= 4;
1569     emit_i8(as, irt_toitype(ir->t));
1570     emit_mrm(as, XO_ARITHi8, XOg_CMP, tmp);
1571     emit_shifti(as, XOg_SAR|REX_64, tmp, 47);
1572     emit_mrm(as, XO_MOV, tmp|REX_64, RID_MRM);
1573   } else if (irt_isnil(ir->t)) {
1574     as->mrm.ofs -= 4;
1575     emit_i8(as, -1);
1576     emit_mrm(as, XO_ARITHi8, XOg_CMP|REX_64, RID_MRM);
1577   } else {
1578     emit_u32(as, (irt_toitype(ir->t) << 15) | 0x7fff);
1579     emit_mrm(as, XO_ARITHi, XOg_CMP, RID_MRM);
1580 #else
1581   } else {
1582     emit_i8(as, irt_toitype(ir->t));
1583     emit_mrm(as, XO_ARITHi8, XOg_CMP, RID_MRM);
1584 #endif
1585   }
1586 }
1587 
1588 static void asm_ahustore(ASMState *as, IRIns *ir)
1589 {
1590   if (ir->r == RID_SINK)
1591     return;
1592   if (irt_isnum(ir->t)) {
1593     Reg src = ra_alloc1(as, ir->op2, RSET_FPR);
1594     asm_fuseahuref(as, ir->op1, RSET_GPR);
1595     emit_mrm(as, XO_MOVSDto, src, RID_MRM);
1596 #if LJ_64 && !LJ_GC64
1597   } else if (irt_islightud(ir->t)) {
1598     Reg src = ra_alloc1(as, ir->op2, RSET_GPR);
1599     asm_fuseahuref(as, ir->op1, rset_exclude(RSET_GPR, src));
1600     emit_mrm(as, XO_MOVto, src|REX_64, RID_MRM);
1601 #endif
1602 #if LJ_GC64
1603   } else if (irref_isk(ir->op2)) {
1604     TValue k;
1605     lj_ir_kvalue(as->J->L, &k, IR(ir->op2));
1606     asm_fuseahuref(as, ir->op1, RSET_GPR);
1607     if (tvisnil(&k)) {
1608       emit_i32(as, -1);
1609       emit_mrm(as, XO_MOVmi, REX_64, RID_MRM);
1610     } else {
1611       emit_u32(as, k.u32.lo);
1612       emit_mrm(as, XO_MOVmi, 0, RID_MRM);
1613       as->mrm.ofs += 4;
1614       emit_u32(as, k.u32.hi);
1615       emit_mrm(as, XO_MOVmi, 0, RID_MRM);
1616     }
1617 #endif
1618   } else {
1619     IRIns *irr = IR(ir->op2);
1620     RegSet allow = RSET_GPR;
1621     Reg src = RID_NONE;
1622     if (!irref_isk(ir->op2)) {
1623       src = ra_alloc1(as, ir->op2, allow);
1624       rset_clear(allow, src);
1625     }
1626     asm_fuseahuref(as, ir->op1, allow);
1627     if (ra_hasreg(src)) {
1628 #if LJ_GC64
1629       if (!(LJ_DUALNUM && irt_isinteger(ir->t))) {
1630         /* TODO: 64 bit store + 32 bit load-modify-store is suboptimal. */
1631         as->mrm.ofs += 4;
1632         emit_u32(as, irt_toitype(ir->t) << 15);
1633         emit_mrm(as, XO_ARITHi, XOg_OR, RID_MRM);
1634         as->mrm.ofs -= 4;
1635         emit_mrm(as, XO_MOVto, src|REX_64, RID_MRM);
1636         return;
1637       }
1638 #endif
1639       emit_mrm(as, XO_MOVto, src, RID_MRM);
1640     } else if (!irt_ispri(irr->t)) {
1641       lua_assert(irt_isaddr(ir->t) || (LJ_DUALNUM && irt_isinteger(ir->t)));
1642       emit_i32(as, irr->i);
1643       emit_mrm(as, XO_MOVmi, 0, RID_MRM);
1644     }
1645     as->mrm.ofs += 4;
1646 #if LJ_GC64
1647     lua_assert(LJ_DUALNUM && irt_isinteger(ir->t));
1648     emit_i32(as, LJ_TNUMX << 15);
1649 #else
1650     emit_i32(as, (int32_t)irt_toitype(ir->t));
1651 #endif
1652     emit_mrm(as, XO_MOVmi, 0, RID_MRM);
1653   }
1654 }
1655 
1656 static void asm_sload(ASMState *as, IRIns *ir)
1657 {
1658   int32_t ofs = 8*((int32_t)ir->op1-1-LJ_FR2) +
1659                 (!LJ_FR2 && (ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
1660   IRType1 t = ir->t;
1661   Reg base;
1662   lua_assert(!(ir->op2 & IRSLOAD_PARENT));  /* Handled by asm_head_side(). */
1663   lua_assert(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK));
1664   lua_assert(LJ_DUALNUM ||
1665              !irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME)));
1666   if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
1667     Reg left = ra_scratch(as, RSET_FPR);
1668     asm_tointg(as, ir, left);  /* Frees dest reg. Do this before base alloc. */
1669     base = ra_alloc1(as, REF_BASE, RSET_GPR);
1670     emit_rmro(as, XO_MOVSD, left, base, ofs);
1671     t.irt = IRT_NUM;  /* Continue with a regular number type check. */
1672 #if LJ_64 && !LJ_GC64
1673   } else if (irt_islightud(t)) {
1674     Reg dest = asm_load_lightud64(as, ir, (ir->op2 & IRSLOAD_TYPECHECK));
1675     if (ra_hasreg(dest)) {
1676       base = ra_alloc1(as, REF_BASE, RSET_GPR);
1677       emit_rmro(as, XO_MOV, dest|REX_64, base, ofs);
1678     }
1679     return;
1680 #endif
1681   } else if (ra_used(ir)) {
1682     RegSet allow = irt_isnum(t) ? RSET_FPR : RSET_GPR;
1683     Reg dest = ra_dest(as, ir, allow);
1684     base = ra_alloc1(as, REF_BASE, RSET_GPR);
1685     lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t));
1686     if ((ir->op2 & IRSLOAD_CONVERT)) {
1687       t.irt = irt_isint(t) ? IRT_NUM : IRT_INT;  /* Check for original type. */
1688       emit_rmro(as, irt_isint(t) ? XO_CVTSI2SD : XO_CVTTSD2SI, dest, base, ofs);
1689     } else {
1690 #if LJ_GC64
1691       if (irt_isaddr(t)) {
1692         /* LJ_GC64 type check + tag removal without BMI2 and with BMI2:
1693         **
1694         **  mov r64, [addr]    rorx r64, [addr], 47
1695         **  ror r64, 47
1696         **  cmp r16, itype     cmp r16, itype
1697         **  jne ->exit         jne ->exit
1698         **  shr r64, 16        shr r64, 16
1699         */
1700         emit_shifti(as, XOg_SHR|REX_64, dest, 17);
1701         if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1702           asm_guardcc(as, CC_NE);
1703           emit_i8(as, irt_toitype(t));
1704           emit_rr(as, XO_ARITHi8, XOg_CMP, dest);
1705           emit_i8(as, XI_O16);
1706         }
1707         if ((as->flags & JIT_F_BMI2)) {
1708           emit_i8(as, 47);
1709           emit_rmro(as, XV_RORX|VEX_64, dest, base, ofs);
1710         } else {
1711           if ((ir->op2 & IRSLOAD_TYPECHECK))
1712             emit_shifti(as, XOg_ROR|REX_64, dest, 47);
1713           else
1714             emit_shifti(as, XOg_SHL|REX_64, dest, 17);
1715           emit_rmro(as, XO_MOV, dest|REX_64, base, ofs);
1716         }
1717         return;
1718       } else
1719 #endif
1720       emit_rmro(as, irt_isnum(t) ? XO_MOVSD : XO_MOV, dest, base, ofs);
1721     }
1722   } else {
1723     if (!(ir->op2 & IRSLOAD_TYPECHECK))
1724       return;  /* No type check: avoid base alloc. */
1725     base = ra_alloc1(as, REF_BASE, RSET_GPR);
1726   }
1727   if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1728     /* Need type check, even if the load result is unused. */
1729     asm_guardcc(as, irt_isnum(t) ? CC_AE : CC_NE);
1730     if (LJ_64 && irt_type(t) >= IRT_NUM) {
1731       lua_assert(irt_isinteger(t) || irt_isnum(t));
1732 #if LJ_GC64
1733       emit_u32(as, LJ_TISNUM << 15);
1734 #else
1735       emit_u32(as, LJ_TISNUM);
1736 #endif
1737       emit_rmro(as, XO_ARITHi, XOg_CMP, base, ofs+4);
1738 #if LJ_GC64
1739     } else if (irt_isnil(t)) {
1740       /* LJ_GC64 type check for nil:
1741       **
1742       **   cmp qword [addr], -1
1743       **   jne ->exit
1744       */
1745       emit_i8(as, -1);
1746       emit_rmro(as, XO_ARITHi8, XOg_CMP|REX_64, base, ofs);
1747     } else if (irt_ispri(t)) {
1748       emit_u32(as, (irt_toitype(t) << 15) | 0x7fff);
1749       emit_rmro(as, XO_ARITHi, XOg_CMP, base, ofs+4);
1750     } else {
1751       /* LJ_GC64 type check only:
1752       **
1753       **   mov r64, [addr]
1754       **   sar r64, 47
1755       **   cmp r32, itype
1756       **   jne ->exit
1757       */
1758       Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, base));
1759       emit_i8(as, irt_toitype(t));
1760       emit_rr(as, XO_ARITHi8, XOg_CMP, tmp);
1761       emit_shifti(as, XOg_SAR|REX_64, tmp, 47);
1762       emit_rmro(as, XO_MOV, tmp|REX_64, base, ofs);
1763 #else
1764     } else {
1765       emit_i8(as, irt_toitype(t));
1766       emit_rmro(as, XO_ARITHi8, XOg_CMP, base, ofs+4);
1767 #endif
1768     }
1769   }
1770 }
1771 
1772 /* -- Allocations --------------------------------------------------------- */
1773 
1774 #if LJ_HASFFI
1775 static void asm_cnew(ASMState *as, IRIns *ir)
1776 {
1777   CTState *cts = ctype_ctsG(J2G(as->J));
1778   CTypeID id = (CTypeID)IR(ir->op1)->i;
1779   CTSize sz;
1780   CTInfo info = lj_ctype_info(cts, id, &sz);
1781   const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1782   IRRef args[4];
1783   lua_assert(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL));
1784 
1785   as->gcsteps++;
1786   asm_setupresult(as, ir, ci);  /* GCcdata * */
1787 
1788   /* Initialize immutable cdata object. */
1789   if (ir->o == IR_CNEWI) {
1790     RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1791 #if LJ_64
1792     Reg r64 = sz == 8 ? REX_64 : 0;
1793     if (irref_isk(ir->op2)) {
1794       IRIns *irk = IR(ir->op2);
1795       uint64_t k = (irk->o == IR_KINT64 ||
1796                     (LJ_GC64 && (irk->o == IR_KPTR || irk->o == IR_KKPTR))) ?
1797                    ir_k64(irk)->u64 : (uint64_t)(uint32_t)irk->i;
1798       if (sz == 4 || checki32((int64_t)k)) {
1799         emit_i32(as, (int32_t)k);
1800         emit_rmro(as, XO_MOVmi, r64, RID_RET, sizeof(GCcdata));
1801       } else {
1802         emit_movtomro(as, RID_ECX + r64, RID_RET, sizeof(GCcdata));
1803         emit_loadu64(as, RID_ECX, k);
1804       }
1805     } else {
1806       Reg r = ra_alloc1(as, ir->op2, allow);
1807       emit_movtomro(as, r + r64, RID_RET, sizeof(GCcdata));
1808     }
1809 #else
1810     int32_t ofs = sizeof(GCcdata);
1811     if (sz == 8) {
1812       ofs += 4; ir++;
1813       lua_assert(ir->o == IR_HIOP);
1814     }
1815     do {
1816       if (irref_isk(ir->op2)) {
1817         emit_movmroi(as, RID_RET, ofs, IR(ir->op2)->i);
1818       } else {
1819         Reg r = ra_alloc1(as, ir->op2, allow);
1820         emit_movtomro(as, r, RID_RET, ofs);
1821         rset_clear(allow, r);
1822       }
1823       if (ofs == sizeof(GCcdata)) break;
1824       ofs -= 4; ir--;
1825     } while (1);
1826 #endif
1827     lua_assert(sz == 4 || sz == 8);
1828   } else if (ir->op2 != REF_NIL) {  /* Create VLA/VLS/aligned cdata. */
1829     ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
1830     args[0] = ASMREF_L;     /* lua_State *L */
1831     args[1] = ir->op1;      /* CTypeID id   */
1832     args[2] = ir->op2;      /* CTSize sz    */
1833     args[3] = ASMREF_TMP1;  /* CTSize align */
1834     asm_gencall(as, ci, args);
1835     emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
1836     return;
1837   }
1838 
1839   /* Combine initialization of marked, gct and ctypeid. */
1840   emit_movtomro(as, RID_ECX, RID_RET, offsetof(GCcdata, marked));
1841   emit_gri(as, XG_ARITHi(XOg_OR), RID_ECX,
1842            (int32_t)((~LJ_TCDATA<<8)+(id<<16)));
1843   emit_gri(as, XG_ARITHi(XOg_AND), RID_ECX, LJ_GC_WHITES);
1844   emit_opgl(as, XO_MOVZXb, RID_ECX, gc.currentwhite);
1845 
1846   args[0] = ASMREF_L;     /* lua_State *L */
1847   args[1] = ASMREF_TMP1;  /* MSize size   */
1848   asm_gencall(as, ci, args);
1849   emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)(sz+sizeof(GCcdata)));
1850 }
1851 #else
1852 #define asm_cnew(as, ir)        ((void)0)
1853 #endif
1854 
1855 /* -- Write barriers ------------------------------------------------------ */
1856 
1857 static void asm_tbar(ASMState *as, IRIns *ir)
1858 {
1859   Reg tab = ra_alloc1(as, ir->op1, RSET_GPR);
1860   Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, tab));
1861   MCLabel l_end = emit_label(as);
1862   emit_movtomro(as, tmp|REX_GC64, tab, offsetof(GCtab, gclist));
1863   emit_setgl(as, tab, gc.grayagain);
1864   emit_getgl(as, tmp, gc.grayagain);
1865   emit_i8(as, ~LJ_GC_BLACK);
1866   emit_rmro(as, XO_ARITHib, XOg_AND, tab, offsetof(GCtab, marked));
1867   emit_sjcc(as, CC_Z, l_end);
1868   emit_i8(as, LJ_GC_BLACK);
1869   emit_rmro(as, XO_GROUP3b, XOg_TEST, tab, offsetof(GCtab, marked));
1870 }
1871 
1872 static void asm_obar(ASMState *as, IRIns *ir)
1873 {
1874   const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_barrieruv];
1875   IRRef args[2];
1876   MCLabel l_end;
1877   Reg obj;
1878   /* No need for other object barriers (yet). */
1879   lua_assert(IR(ir->op1)->o == IR_UREFC);
1880   ra_evictset(as, RSET_SCRATCH);
1881   l_end = emit_label(as);
1882   args[0] = ASMREF_TMP1;  /* global_State *g */
1883   args[1] = ir->op1;      /* TValue *tv      */
1884   asm_gencall(as, ci, args);
1885   emit_loada(as, ra_releasetmp(as, ASMREF_TMP1), J2G(as->J));
1886   obj = IR(ir->op1)->r;
1887   emit_sjcc(as, CC_Z, l_end);
1888   emit_i8(as, LJ_GC_WHITES);
1889   if (irref_isk(ir->op2)) {
1890     GCobj *vp = ir_kgc(IR(ir->op2));
1891     emit_rma(as, XO_GROUP3b, XOg_TEST, &vp->gch.marked);
1892   } else {
1893     Reg val = ra_alloc1(as, ir->op2, rset_exclude(RSET_SCRATCH&RSET_GPR, obj));
1894     emit_rmro(as, XO_GROUP3b, XOg_TEST, val, (int32_t)offsetof(GChead, marked));
1895   }
1896   emit_sjcc(as, CC_Z, l_end);
1897   emit_i8(as, LJ_GC_BLACK);
1898   emit_rmro(as, XO_GROUP3b, XOg_TEST, obj,
1899             (int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv));
1900 }
1901 
1902 /* -- FP/int arithmetic and logic operations ------------------------------ */
1903 
1904 /* Load reference onto x87 stack. Force a spill to memory if needed. */
1905 static void asm_x87load(ASMState *as, IRRef ref)
1906 {
1907   IRIns *ir = IR(ref);
1908   if (ir->o == IR_KNUM) {
1909     cTValue *tv = ir_knum(ir);
1910     if (tvispzero(tv))  /* Use fldz only for +0. */
1911       emit_x87op(as, XI_FLDZ);
1912     else if (tvispone(tv))
1913       emit_x87op(as, XI_FLD1);
1914     else
1915       emit_rma(as, XO_FLDq, XOg_FLDq, tv);
1916   } else if (ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT && !ra_used(ir) &&
1917              !irref_isk(ir->op1) && mayfuse(as, ir->op1)) {
1918     IRIns *iri = IR(ir->op1);
1919     emit_rmro(as, XO_FILDd, XOg_FILDd, RID_ESP, ra_spill(as, iri));
1920   } else {
1921     emit_mrm(as, XO_FLDq, XOg_FLDq, asm_fuseload(as, ref, RSET_EMPTY));
1922   }
1923 }
1924 
1925 static void asm_fpmath(ASMState *as, IRIns *ir)
1926 {
1927   IRFPMathOp fpm = (IRFPMathOp)ir->op2;
1928   if (fpm == IRFPM_SQRT) {
1929     Reg dest = ra_dest(as, ir, RSET_FPR);
1930     Reg left = asm_fuseload(as, ir->op1, RSET_FPR);
1931     emit_mrm(as, XO_SQRTSD, dest, left);
1932   } else if (fpm <= IRFPM_TRUNC) {
1933     if (as->flags & JIT_F_SSE4_1) {  /* SSE4.1 has a rounding instruction. */
1934       Reg dest = ra_dest(as, ir, RSET_FPR);
1935       Reg left = asm_fuseload(as, ir->op1, RSET_FPR);
1936       /* ROUNDSD has a 4-byte opcode which doesn't fit in x86Op.
1937       ** Let's pretend it's a 3-byte opcode, and compensate afterwards.
1938       ** This is atrocious, but the alternatives are much worse.
1939       */
1940       /* Round down/up/trunc == 1001/1010/1011. */
1941       emit_i8(as, 0x09 + fpm);
1942       emit_mrm(as, XO_ROUNDSD, dest, left);
1943       if (LJ_64 && as->mcp[1] != (MCode)(XO_ROUNDSD >> 16)) {
1944         as->mcp[0] = as->mcp[1]; as->mcp[1] = 0x0f;  /* Swap 0F and REX. */
1945       }
1946       *--as->mcp = 0x66;  /* 1st byte of ROUNDSD opcode. */
1947     } else {  /* Call helper functions for SSE2 variant. */
1948       /* The modified regs must match with the *.dasc implementation. */
1949       RegSet drop = RSET_RANGE(RID_XMM0, RID_XMM3+1)|RID2RSET(RID_EAX);
1950       if (ra_hasreg(ir->r))
1951         rset_clear(drop, ir->r);  /* Dest reg handled below. */
1952       ra_evictset(as, drop);
1953       ra_destreg(as, ir, RID_XMM0);
1954       emit_call(as, fpm == IRFPM_FLOOR ? lj_vm_floor_sse :
1955                     fpm == IRFPM_CEIL ? lj_vm_ceil_sse : lj_vm_trunc_sse);
1956       ra_left(as, RID_XMM0, ir->op1);
1957     }
1958   } else if (fpm == IRFPM_EXP2 && asm_fpjoin_pow(as, ir)) {
1959     /* Rejoined to pow(). */
1960   } else {
1961     asm_callid(as, ir, IRCALL_lj_vm_floor + fpm);
1962   }
1963 }
1964 
1965 #define asm_atan2(as, ir)       asm_callid(as, ir, IRCALL_atan2)
1966 
1967 static void asm_ldexp(ASMState *as, IRIns *ir)
1968 {
1969   int32_t ofs = sps_scale(ir->s);  /* Use spill slot or temp slots. */
1970   Reg dest = ir->r;
1971   if (ra_hasreg(dest)) {
1972     ra_free(as, dest);
1973     ra_modified(as, dest);
1974     emit_rmro(as, XO_MOVSD, dest, RID_ESP, ofs);
1975   }
1976   emit_rmro(as, XO_FSTPq, XOg_FSTPq, RID_ESP, ofs);
1977   emit_x87op(as, XI_FPOP1);
1978   emit_x87op(as, XI_FSCALE);
1979   asm_x87load(as, ir->op1);
1980   asm_x87load(as, ir->op2);
1981 }
1982 
1983 static void asm_fppowi(ASMState *as, IRIns *ir)
1984 {
1985   /* The modified regs must match with the *.dasc implementation. */
1986   RegSet drop = RSET_RANGE(RID_XMM0, RID_XMM1+1)|RID2RSET(RID_EAX);
1987   if (ra_hasreg(ir->r))
1988     rset_clear(drop, ir->r);  /* Dest reg handled below. */
1989   ra_evictset(as, drop);
1990   ra_destreg(as, ir, RID_XMM0);
1991   emit_call(as, lj_vm_powi_sse);
1992   ra_left(as, RID_XMM0, ir->op1);
1993   ra_left(as, RID_EAX, ir->op2);
1994 }
1995 
1996 static void asm_pow(ASMState *as, IRIns *ir)
1997 {
1998 #if LJ_64 && LJ_HASFFI
1999   if (!irt_isnum(ir->t))
2000     asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
2001                                           IRCALL_lj_carith_powu64);
2002   else
2003 #endif
2004     asm_fppowi(as, ir);
2005 }
2006 
2007 static int asm_swapops(ASMState *as, IRIns *ir)
2008 {
2009   IRIns *irl = IR(ir->op1);
2010   IRIns *irr = IR(ir->op2);
2011   lua_assert(ra_noreg(irr->r));
2012   if (!irm_iscomm(lj_ir_mode[ir->o]))
2013     return 0;  /* Can't swap non-commutative operations. */
2014   if (irref_isk(ir->op2))
2015     return 0;  /* Don't swap constants to the left. */
2016   if (ra_hasreg(irl->r))
2017     return 1;  /* Swap if left already has a register. */
2018   if (ra_samehint(ir->r, irr->r))
2019     return 1;  /* Swap if dest and right have matching hints. */
2020   if (as->curins > as->loopref) {  /* In variant part? */
2021     if (ir->op2 < as->loopref && !irt_isphi(irr->t))
2022       return 0;  /* Keep invariants on the right. */
2023     if (ir->op1 < as->loopref && !irt_isphi(irl->t))
2024       return 1;  /* Swap invariants to the right. */
2025   }
2026   if (opisfusableload(irl->o))
2027     return 1;  /* Swap fusable loads to the right. */
2028   return 0;  /* Otherwise don't swap. */
2029 }
2030 
2031 static void asm_fparith(ASMState *as, IRIns *ir, x86Op xo)
2032 {
2033   IRRef lref = ir->op1;
2034   IRRef rref = ir->op2;
2035   RegSet allow = RSET_FPR;
2036   Reg dest;
2037   Reg right = IR(rref)->r;
2038   if (ra_hasreg(right)) {
2039     rset_clear(allow, right);
2040     ra_noweak(as, right);
2041   }
2042   dest = ra_dest(as, ir, allow);
2043   if (lref == rref) {
2044     right = dest;
2045   } else if (ra_noreg(right)) {
2046     if (asm_swapops(as, ir)) {
2047       IRRef tmp = lref; lref = rref; rref = tmp;
2048     }
2049     right = asm_fuseload(as, rref, rset_clear(allow, dest));
2050   }
2051   emit_mrm(as, xo, dest, right);
2052   ra_left(as, dest, lref);
2053 }
2054 
2055 static void asm_intarith(ASMState *as, IRIns *ir, x86Arith xa)
2056 {
2057   IRRef lref = ir->op1;
2058   IRRef rref = ir->op2;
2059   RegSet allow = RSET_GPR;
2060   Reg dest, right;
2061   int32_t k = 0;
2062   if (as->flagmcp == as->mcp) {  /* Drop test r,r instruction. */
2063     MCode *p = as->mcp + ((LJ_64 && *as->mcp < XI_TESTb) ? 3 : 2);
2064     if ((p[1] & 15) < 14) {
2065       if ((p[1] & 15) >= 12) p[1] -= 4;  /* L <->S, NL <-> NS */
2066       as->flagmcp = NULL;
2067       as->mcp = p;
2068     }  /* else: cannot transform LE/NLE to cc without use of OF. */
2069   }
2070   right = IR(rref)->r;
2071   if (ra_hasreg(right)) {
2072     rset_clear(allow, right);
2073     ra_noweak(as, right);
2074   }
2075   dest = ra_dest(as, ir, allow);
2076   if (lref == rref) {
2077     right = dest;
2078   } else if (ra_noreg(right) && !asm_isk32(as, rref, &k)) {
2079     if (asm_swapops(as, ir)) {
2080       IRRef tmp = lref; lref = rref; rref = tmp;
2081     }
2082     right = asm_fuseloadm(as, rref, rset_clear(allow, dest), irt_is64(ir->t));
2083   }
2084   if (irt_isguard(ir->t))  /* For IR_ADDOV etc. */
2085     asm_guardcc(as, CC_O);
2086   if (xa != XOg_X_IMUL) {
2087     if (ra_hasreg(right))
2088       emit_mrm(as, XO_ARITH(xa), REX_64IR(ir, dest), right);
2089     else
2090       emit_gri(as, XG_ARITHi(xa), REX_64IR(ir, dest), k);
2091   } else if (ra_hasreg(right)) {  /* IMUL r, mrm. */
2092     emit_mrm(as, XO_IMUL, REX_64IR(ir, dest), right);
2093   } else {  /* IMUL r, r, k. */
2094     /* NYI: use lea/shl/add/sub (FOLD only does 2^k) depending on CPU. */
2095     Reg left = asm_fuseloadm(as, lref, RSET_GPR, irt_is64(ir->t));
2096     x86Op xo;
2097     if (checki8(k)) { emit_i8(as, k); xo = XO_IMULi8;
2098     } else { emit_i32(as, k); xo = XO_IMULi; }
2099     emit_mrm(as, xo, REX_64IR(ir, dest), left);
2100     return;
2101   }
2102   ra_left(as, dest, lref);
2103 }
2104 
2105 /* LEA is really a 4-operand ADD with an independent destination register,
2106 ** up to two source registers and an immediate. One register can be scaled
2107 ** by 1, 2, 4 or 8. This can be used to avoid moves or to fuse several
2108 ** instructions.
2109 **
2110 ** Currently only a few common cases are supported:
2111 ** - 3-operand ADD:    y = a+b; y = a+k   with a and b already allocated
2112 ** - Left ADD fusion:  y = (a+b)+k; y = (a+k)+b
2113 ** - Right ADD fusion: y = a+(b+k)
2114 ** The ommited variants have already been reduced by FOLD.
2115 **
2116 ** There are more fusion opportunities, like gathering shifts or joining
2117 ** common references. But these are probably not worth the trouble, since
2118 ** array indexing is not decomposed and already makes use of all fields
2119 ** of the ModRM operand.
2120 */
2121 static int asm_lea(ASMState *as, IRIns *ir)
2122 {
2123   IRIns *irl = IR(ir->op1);
2124   IRIns *irr = IR(ir->op2);
2125   RegSet allow = RSET_GPR;
2126   Reg dest;
2127   as->mrm.base = as->mrm.idx = RID_NONE;
2128   as->mrm.scale = XM_SCALE1;
2129   as->mrm.ofs = 0;
2130   if (ra_hasreg(irl->r)) {
2131     rset_clear(allow, irl->r);
2132     ra_noweak(as, irl->r);
2133     as->mrm.base = irl->r;
2134     if (irref_isk(ir->op2) || ra_hasreg(irr->r)) {
2135       /* The PHI renaming logic does a better job in some cases. */
2136       if (ra_hasreg(ir->r) &&
2137           ((irt_isphi(irl->t) && as->phireg[ir->r] == ir->op1) ||
2138            (irt_isphi(irr->t) && as->phireg[ir->r] == ir->op2)))
2139         return 0;
2140       if (irref_isk(ir->op2)) {
2141         as->mrm.ofs = irr->i;
2142       } else {
2143         rset_clear(allow, irr->r);
2144         ra_noweak(as, irr->r);
2145         as->mrm.idx = irr->r;
2146       }
2147     } else if (irr->o == IR_ADD && mayfuse(as, ir->op2) &&
2148                irref_isk(irr->op2)) {
2149       Reg idx = ra_alloc1(as, irr->op1, allow);
2150       rset_clear(allow, idx);
2151       as->mrm.idx = (uint8_t)idx;
2152       as->mrm.ofs = IR(irr->op2)->i;
2153     } else {
2154       return 0;
2155     }
2156   } else if (ir->op1 != ir->op2 && irl->o == IR_ADD && mayfuse(as, ir->op1) &&
2157              (irref_isk(ir->op2) || irref_isk(irl->op2))) {
2158     Reg idx, base = ra_alloc1(as, irl->op1, allow);
2159     rset_clear(allow, base);
2160     as->mrm.base = (uint8_t)base;
2161     if (irref_isk(ir->op2)) {
2162       as->mrm.ofs = irr->i;
2163       idx = ra_alloc1(as, irl->op2, allow);
2164     } else {
2165       as->mrm.ofs = IR(irl->op2)->i;
2166       idx = ra_alloc1(as, ir->op2, allow);
2167     }
2168     rset_clear(allow, idx);
2169     as->mrm.idx = (uint8_t)idx;
2170   } else {
2171     return 0;
2172   }
2173   dest = ra_dest(as, ir, allow);
2174   emit_mrm(as, XO_LEA, dest, RID_MRM);
2175   return 1;  /* Success. */
2176 }
2177 
2178 static void asm_add(ASMState *as, IRIns *ir)
2179 {
2180   if (irt_isnum(ir->t))
2181     asm_fparith(as, ir, XO_ADDSD);
2182   else if ((as->flags & JIT_F_LEA_AGU) || as->flagmcp == as->mcp ||
2183            irt_is64(ir->t) || !asm_lea(as, ir))
2184     asm_intarith(as, ir, XOg_ADD);
2185 }
2186 
2187 static void asm_sub(ASMState *as, IRIns *ir)
2188 {
2189   if (irt_isnum(ir->t))
2190     asm_fparith(as, ir, XO_SUBSD);
2191   else  /* Note: no need for LEA trick here. i-k is encoded as i+(-k). */
2192     asm_intarith(as, ir, XOg_SUB);
2193 }
2194 
2195 static void asm_mul(ASMState *as, IRIns *ir)
2196 {
2197   if (irt_isnum(ir->t))
2198     asm_fparith(as, ir, XO_MULSD);
2199   else
2200     asm_intarith(as, ir, XOg_X_IMUL);
2201 }
2202 
2203 static void asm_div(ASMState *as, IRIns *ir)
2204 {
2205 #if LJ_64 && LJ_HASFFI
2206   if (!irt_isnum(ir->t))
2207     asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
2208                                           IRCALL_lj_carith_divu64);
2209   else
2210 #endif
2211     asm_fparith(as, ir, XO_DIVSD);
2212 }
2213 
2214 static void asm_mod(ASMState *as, IRIns *ir)
2215 {
2216 #if LJ_64 && LJ_HASFFI
2217   if (!irt_isint(ir->t))
2218     asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
2219                                           IRCALL_lj_carith_modu64);
2220   else
2221 #endif
2222     asm_callid(as, ir, IRCALL_lj_vm_modi);
2223 }
2224 
2225 static void asm_neg_not(ASMState *as, IRIns *ir, x86Group3 xg)
2226 {
2227   Reg dest = ra_dest(as, ir, RSET_GPR);
2228   emit_rr(as, XO_GROUP3, REX_64IR(ir, xg), dest);
2229   ra_left(as, dest, ir->op1);
2230 }
2231 
2232 static void asm_neg(ASMState *as, IRIns *ir)
2233 {
2234   if (irt_isnum(ir->t))
2235     asm_fparith(as, ir, XO_XORPS);
2236   else
2237     asm_neg_not(as, ir, XOg_NEG);
2238 }
2239 
2240 #define asm_abs(as, ir)         asm_fparith(as, ir, XO_ANDPS)
2241 
2242 static void asm_intmin_max(ASMState *as, IRIns *ir, int cc)
2243 {
2244   Reg right, dest = ra_dest(as, ir, RSET_GPR);
2245   IRRef lref = ir->op1, rref = ir->op2;
2246   if (irref_isk(rref)) { lref = rref; rref = ir->op1; }
2247   right = ra_alloc1(as, rref, rset_exclude(RSET_GPR, dest));
2248   emit_rr(as, XO_CMOV + (cc<<24), REX_64IR(ir, dest), right);
2249   emit_rr(as, XO_CMP, REX_64IR(ir, dest), right);
2250   ra_left(as, dest, lref);
2251 }
2252 
2253 static void asm_min(ASMState *as, IRIns *ir)
2254 {
2255   if (irt_isnum(ir->t))
2256     asm_fparith(as, ir, XO_MINSD);
2257   else
2258     asm_intmin_max(as, ir, CC_G);
2259 }
2260 
2261 static void asm_max(ASMState *as, IRIns *ir)
2262 {
2263   if (irt_isnum(ir->t))
2264     asm_fparith(as, ir, XO_MAXSD);
2265   else
2266     asm_intmin_max(as, ir, CC_L);
2267 }
2268 
2269 /* Note: don't use LEA for overflow-checking arithmetic! */
2270 #define asm_addov(as, ir)       asm_intarith(as, ir, XOg_ADD)
2271 #define asm_subov(as, ir)       asm_intarith(as, ir, XOg_SUB)
2272 #define asm_mulov(as, ir)       asm_intarith(as, ir, XOg_X_IMUL)
2273 
2274 #define asm_bnot(as, ir)        asm_neg_not(as, ir, XOg_NOT)
2275 
2276 static void asm_bswap(ASMState *as, IRIns *ir)
2277 {
2278   Reg dest = ra_dest(as, ir, RSET_GPR);
2279   as->mcp = emit_op(XO_BSWAP + ((dest&7) << 24),
2280                     REX_64IR(ir, 0), dest, 0, as->mcp, 1);
2281   ra_left(as, dest, ir->op1);
2282 }
2283 
2284 #define asm_band(as, ir)        asm_intarith(as, ir, XOg_AND)
2285 #define asm_bor(as, ir)         asm_intarith(as, ir, XOg_OR)
2286 #define asm_bxor(as, ir)        asm_intarith(as, ir, XOg_XOR)
2287 
2288 static void asm_bitshift(ASMState *as, IRIns *ir, x86Shift xs, x86Op xv)
2289 {
2290   IRRef rref = ir->op2;
2291   IRIns *irr = IR(rref);
2292   Reg dest;
2293   if (irref_isk(rref)) {  /* Constant shifts. */
2294     int shift;
2295     dest = ra_dest(as, ir, RSET_GPR);
2296     shift = irr->i & (irt_is64(ir->t) ? 63 : 31);
2297     if (!xv && shift && (as->flags & JIT_F_BMI2)) {
2298       Reg left = asm_fuseloadm(as, ir->op1, RSET_GPR, irt_is64(ir->t));
2299       if (left != dest) {  /* BMI2 rotate right by constant. */
2300         emit_i8(as, xs == XOg_ROL ? -shift : shift);
2301         emit_mrm(as, VEX_64IR(ir, XV_RORX), dest, left);
2302         return;
2303       }
2304     }
2305     switch (shift) {
2306     case 0: break;
2307     case 1: emit_rr(as, XO_SHIFT1, REX_64IR(ir, xs), dest); break;
2308     default: emit_shifti(as, REX_64IR(ir, xs), dest, shift); break;
2309     }
2310   } else if ((as->flags & JIT_F_BMI2) && xv) {  /* BMI2 variable shifts. */
2311     Reg left, right;
2312     dest = ra_dest(as, ir, RSET_GPR);
2313     right = ra_alloc1(as, rref, RSET_GPR);
2314     left = asm_fuseloadm(as, ir->op1, rset_exclude(RSET_GPR, right),
2315                          irt_is64(ir->t));
2316     emit_mrm(as, VEX_64IR(ir, xv) ^ (right << 19), dest, left);
2317     return;
2318   } else {  /* Variable shifts implicitly use register cl (i.e. ecx). */
2319     Reg right;
2320     dest = ra_dest(as, ir, rset_exclude(RSET_GPR, RID_ECX));
2321     if (dest == RID_ECX) {
2322       dest = ra_scratch(as, rset_exclude(RSET_GPR, RID_ECX));
2323       emit_rr(as, XO_MOV, RID_ECX, dest);
2324     }
2325     right = irr->r;
2326     if (ra_noreg(right))
2327       right = ra_allocref(as, rref, RID2RSET(RID_ECX));
2328     else if (right != RID_ECX)
2329       ra_scratch(as, RID2RSET(RID_ECX));
2330     emit_rr(as, XO_SHIFTcl, REX_64IR(ir, xs), dest);
2331     ra_noweak(as, right);
2332     if (right != RID_ECX)
2333       emit_rr(as, XO_MOV, RID_ECX, right);
2334   }
2335   ra_left(as, dest, ir->op1);
2336   /*
2337   ** Note: avoid using the flags resulting from a shift or rotate!
2338   ** All of them cause a partial flag stall, except for r,1 shifts
2339   ** (but not rotates). And a shift count of 0 leaves the flags unmodified.
2340   */
2341 }
2342 
2343 #define asm_bshl(as, ir)        asm_bitshift(as, ir, XOg_SHL, XV_SHLX)
2344 #define asm_bshr(as, ir)        asm_bitshift(as, ir, XOg_SHR, XV_SHRX)
2345 #define asm_bsar(as, ir)        asm_bitshift(as, ir, XOg_SAR, XV_SARX)
2346 #define asm_brol(as, ir)        asm_bitshift(as, ir, XOg_ROL, 0)
2347 #define asm_bror(as, ir)        asm_bitshift(as, ir, XOg_ROR, 0)
2348 
2349 /* -- Comparisons --------------------------------------------------------- */
2350 
2351 /* Virtual flags for unordered FP comparisons. */
2352 #define VCC_U   0x1000          /* Unordered. */
2353 #define VCC_P   0x2000          /* Needs extra CC_P branch. */
2354 #define VCC_S   0x4000          /* Swap avoids CC_P branch. */
2355 #define VCC_PS  (VCC_P|VCC_S)
2356 
2357 /* Map of comparisons to flags. ORDER IR. */
2358 #define COMPFLAGS(ci, cin, cu, cf)      ((ci)+((cu)<<4)+((cin)<<8)+(cf))
2359 static const uint16_t asm_compmap[IR_ABC+1] = {
2360   /*                 signed non-eq unsigned flags */
2361   /* LT  */ COMPFLAGS(CC_GE, CC_G,  CC_AE, VCC_PS),
2362   /* GE  */ COMPFLAGS(CC_L,  CC_L,  CC_B,  0),
2363   /* LE  */ COMPFLAGS(CC_G,  CC_G,  CC_A,  VCC_PS),
2364   /* GT  */ COMPFLAGS(CC_LE, CC_L,  CC_BE, 0),
2365   /* ULT */ COMPFLAGS(CC_AE, CC_A,  CC_AE, VCC_U),
2366   /* UGE */ COMPFLAGS(CC_B,  CC_B,  CC_B,  VCC_U|VCC_PS),
2367   /* ULE */ COMPFLAGS(CC_A,  CC_A,  CC_A,  VCC_U),
2368   /* UGT */ COMPFLAGS(CC_BE, CC_B,  CC_BE, VCC_U|VCC_PS),
2369   /* EQ  */ COMPFLAGS(CC_NE, CC_NE, CC_NE, VCC_P),
2370   /* NE  */ COMPFLAGS(CC_E,  CC_E,  CC_E,  VCC_U|VCC_P),
2371   /* ABC */ COMPFLAGS(CC_BE, CC_B,  CC_BE, VCC_U|VCC_PS)  /* Same as UGT. */
2372 };
2373 
2374 /* FP and integer comparisons. */
2375 static void asm_comp(ASMState *as, IRIns *ir)
2376 {
2377   uint32_t cc = asm_compmap[ir->o];
2378   if (irt_isnum(ir->t)) {
2379     IRRef lref = ir->op1;
2380     IRRef rref = ir->op2;
2381     Reg left, right;
2382     MCLabel l_around;
2383     /*
2384     ** An extra CC_P branch is required to preserve ordered/unordered
2385     ** semantics for FP comparisons. This can be avoided by swapping
2386     ** the operands and inverting the condition (except for EQ and UNE).
2387     ** So always try to swap if possible.
2388     **
2389     ** Another option would be to swap operands to achieve better memory
2390     ** operand fusion. But it's unlikely that this outweighs the cost
2391     ** of the extra branches.
2392     */
2393     if (cc & VCC_S) {  /* Swap? */
2394       IRRef tmp = lref; lref = rref; rref = tmp;
2395       cc ^= (VCC_PS|(5<<4));  /* A <-> B, AE <-> BE, PS <-> none */
2396     }
2397     left = ra_alloc1(as, lref, RSET_FPR);
2398     l_around = emit_label(as);
2399     asm_guardcc(as, cc >> 4);
2400     if (cc & VCC_P) {  /* Extra CC_P branch required? */
2401       if (!(cc & VCC_U)) {
2402         asm_guardcc(as, CC_P);  /* Branch to exit for ordered comparisons. */
2403       } else if (l_around != as->invmcp) {
2404         emit_sjcc(as, CC_P, l_around);  /* Branch around for unordered. */
2405       } else {
2406         /* Patched to mcloop by asm_loop_fixup. */
2407         as->loopinv = 2;
2408         if (as->realign)
2409           emit_sjcc(as, CC_P, as->mcp);
2410         else
2411           emit_jcc(as, CC_P, as->mcp);
2412       }
2413     }
2414     right = asm_fuseload(as, rref, rset_exclude(RSET_FPR, left));
2415     emit_mrm(as, XO_UCOMISD, left, right);
2416   } else {
2417     IRRef lref = ir->op1, rref = ir->op2;
2418     IROp leftop = (IROp)(IR(lref)->o);
2419     Reg r64 = REX_64IR(ir, 0);
2420     int32_t imm = 0;
2421     lua_assert(irt_is64(ir->t) || irt_isint(ir->t) ||
2422                irt_isu32(ir->t) || irt_isaddr(ir->t) || irt_isu8(ir->t));
2423     /* Swap constants (only for ABC) and fusable loads to the right. */
2424     if (irref_isk(lref) || (!irref_isk(rref) && opisfusableload(leftop))) {
2425       if ((cc & 0xc) == 0xc) cc ^= 0x53;  /* L <-> G, LE <-> GE */
2426       else if ((cc & 0xa) == 0x2) cc ^= 0x55;  /* A <-> B, AE <-> BE */
2427       lref = ir->op2; rref = ir->op1;
2428     }
2429     if (asm_isk32(as, rref, &imm)) {
2430       IRIns *irl = IR(lref);
2431       /* Check wether we can use test ins. Not for unsigned, since CF=0. */
2432       int usetest = (imm == 0 && (cc & 0xa) != 0x2);
2433       if (usetest && irl->o == IR_BAND && irl+1 == ir && !ra_used(irl)) {
2434         /* Combine comp(BAND(ref, r/imm), 0) into test mrm, r/imm. */
2435         Reg right, left = RID_NONE;
2436         RegSet allow = RSET_GPR;
2437         if (!asm_isk32(as, irl->op2, &imm)) {
2438           left = ra_alloc1(as, irl->op2, allow);
2439           rset_clear(allow, left);
2440         } else {  /* Try to Fuse IRT_I8/IRT_U8 loads, too. See below. */
2441           IRIns *irll = IR(irl->op1);
2442           if (opisfusableload((IROp)irll->o) &&
2443               (irt_isi8(irll->t) || irt_isu8(irll->t))) {
2444             IRType1 origt = irll->t;  /* Temporarily flip types. */
2445             irll->t.irt = (irll->t.irt & ~IRT_TYPE) | IRT_INT;
2446             as->curins--;  /* Skip to BAND to avoid failing in noconflict(). */
2447             right = asm_fuseload(as, irl->op1, RSET_GPR);
2448             as->curins++;
2449             irll->t = origt;
2450             if (right != RID_MRM) goto test_nofuse;
2451             /* Fusion succeeded, emit test byte mrm, imm8. */
2452             asm_guardcc(as, cc);
2453             emit_i8(as, (imm & 0xff));
2454             emit_mrm(as, XO_GROUP3b, XOg_TEST, RID_MRM);
2455             return;
2456           }
2457         }
2458         as->curins--;  /* Skip to BAND to avoid failing in noconflict(). */
2459         right = asm_fuseloadm(as, irl->op1, allow, r64);
2460         as->curins++;  /* Undo the above. */
2461       test_nofuse:
2462         asm_guardcc(as, cc);
2463         if (ra_noreg(left)) {
2464           emit_i32(as, imm);
2465           emit_mrm(as, XO_GROUP3, r64 + XOg_TEST, right);
2466         } else {
2467           emit_mrm(as, XO_TEST, r64 + left, right);
2468         }
2469       } else {
2470         Reg left;
2471         if (opisfusableload((IROp)irl->o) &&
2472             ((irt_isu8(irl->t) && checku8(imm)) ||
2473              ((irt_isi8(irl->t) || irt_isi16(irl->t)) && checki8(imm)) ||
2474              (irt_isu16(irl->t) && checku16(imm) && checki8((int16_t)imm)))) {
2475           /* Only the IRT_INT case is fused by asm_fuseload.
2476           ** The IRT_I8/IRT_U8 loads and some IRT_I16/IRT_U16 loads
2477           ** are handled here.
2478           ** Note that cmp word [mem], imm16 should not be generated,
2479           ** since it has a length-changing prefix. Compares of a word
2480           ** against a sign-extended imm8 are ok, however.
2481           */
2482           IRType1 origt = irl->t;  /* Temporarily flip types. */
2483           irl->t.irt = (irl->t.irt & ~IRT_TYPE) | IRT_INT;
2484           left = asm_fuseload(as, lref, RSET_GPR);
2485           irl->t = origt;
2486           if (left == RID_MRM) {  /* Fusion succeeded? */
2487             if (irt_isu8(irl->t) || irt_isu16(irl->t))
2488               cc >>= 4;  /* Need unsigned compare. */
2489             asm_guardcc(as, cc);
2490             emit_i8(as, imm);
2491             emit_mrm(as, (irt_isi8(origt) || irt_isu8(origt)) ?
2492                          XO_ARITHib : XO_ARITHiw8, r64 + XOg_CMP, RID_MRM);
2493             return;
2494           }  /* Otherwise handle register case as usual. */
2495         } else {
2496           left = asm_fuseloadm(as, lref,
2497                                irt_isu8(ir->t) ? RSET_GPR8 : RSET_GPR, r64);
2498         }
2499         asm_guardcc(as, cc);
2500         if (usetest && left != RID_MRM) {
2501           /* Use test r,r instead of cmp r,0. */
2502           x86Op xo = XO_TEST;
2503           if (irt_isu8(ir->t)) {
2504             lua_assert(ir->o == IR_EQ || ir->o == IR_NE);
2505             xo = XO_TESTb;
2506             if (!rset_test(RSET_RANGE(RID_EAX, RID_EBX+1), left)) {
2507               if (LJ_64) {
2508                 left |= FORCE_REX;
2509               } else {
2510                 emit_i32(as, 0xff);
2511                 emit_mrm(as, XO_GROUP3, XOg_TEST, left);
2512                 return;
2513               }
2514             }
2515           }
2516           emit_rr(as, xo, r64 + left, left);
2517           if (irl+1 == ir)  /* Referencing previous ins? */
2518             as->flagmcp = as->mcp;  /* Set flag to drop test r,r if possible. */
2519         } else {
2520           emit_gmrmi(as, XG_ARITHi(XOg_CMP), r64 + left, imm);
2521         }
2522       }
2523     } else {
2524       Reg left = ra_alloc1(as, lref, RSET_GPR);
2525       Reg right = asm_fuseloadm(as, rref, rset_exclude(RSET_GPR, left), r64);
2526       asm_guardcc(as, cc);
2527       emit_mrm(as, XO_CMP, r64 + left, right);
2528     }
2529   }
2530 }
2531 
2532 #define asm_equal(as, ir)       asm_comp(as, ir)
2533 
2534 #if LJ_32 && LJ_HASFFI
2535 /* 64 bit integer comparisons in 32 bit mode. */
2536 static void asm_comp_int64(ASMState *as, IRIns *ir)
2537 {
2538   uint32_t cc = asm_compmap[(ir-1)->o];
2539   RegSet allow = RSET_GPR;
2540   Reg lefthi = RID_NONE, leftlo = RID_NONE;
2541   Reg righthi = RID_NONE, rightlo = RID_NONE;
2542   MCLabel l_around;
2543   x86ModRM mrm;
2544 
2545   as->curins--;  /* Skip loword ins. Avoids failing in noconflict(), too. */
2546 
2547   /* Allocate/fuse hiword operands. */
2548   if (irref_isk(ir->op2)) {
2549     lefthi = asm_fuseload(as, ir->op1, allow);
2550   } else {
2551     lefthi = ra_alloc1(as, ir->op1, allow);
2552     rset_clear(allow, lefthi);
2553     righthi = asm_fuseload(as, ir->op2, allow);
2554     if (righthi == RID_MRM) {
2555       if (as->mrm.base != RID_NONE) rset_clear(allow, as->mrm.base);
2556       if (as->mrm.idx != RID_NONE) rset_clear(allow, as->mrm.idx);
2557     } else {
2558       rset_clear(allow, righthi);
2559     }
2560   }
2561   mrm = as->mrm;  /* Save state for hiword instruction. */
2562 
2563   /* Allocate/fuse loword operands. */
2564   if (irref_isk((ir-1)->op2)) {
2565     leftlo = asm_fuseload(as, (ir-1)->op1, allow);
2566   } else {
2567     leftlo = ra_alloc1(as, (ir-1)->op1, allow);
2568     rset_clear(allow, leftlo);
2569     rightlo = asm_fuseload(as, (ir-1)->op2, allow);
2570   }
2571 
2572   /* All register allocations must be performed _before_ this point. */
2573   l_around = emit_label(as);
2574   as->invmcp = as->flagmcp = NULL;  /* Cannot use these optimizations. */
2575 
2576   /* Loword comparison and branch. */
2577   asm_guardcc(as, cc >> 4);  /* Always use unsigned compare for loword. */
2578   if (ra_noreg(rightlo)) {
2579     int32_t imm = IR((ir-1)->op2)->i;
2580     if (imm == 0 && ((cc >> 4) & 0xa) != 0x2 && leftlo != RID_MRM)
2581       emit_rr(as, XO_TEST, leftlo, leftlo);
2582     else
2583       emit_gmrmi(as, XG_ARITHi(XOg_CMP), leftlo, imm);
2584   } else {
2585     emit_mrm(as, XO_CMP, leftlo, rightlo);
2586   }
2587 
2588   /* Hiword comparison and branches. */
2589   if ((cc & 15) != CC_NE)
2590     emit_sjcc(as, CC_NE, l_around);  /* Hiword unequal: skip loword compare. */
2591   if ((cc & 15) != CC_E)
2592     asm_guardcc(as, cc >> 8);  /* Hiword compare without equality check. */
2593   as->mrm = mrm;  /* Restore state. */
2594   if (ra_noreg(righthi)) {
2595     int32_t imm = IR(ir->op2)->i;
2596     if (imm == 0 && (cc & 0xa) != 0x2 && lefthi != RID_MRM)
2597       emit_rr(as, XO_TEST, lefthi, lefthi);
2598     else
2599       emit_gmrmi(as, XG_ARITHi(XOg_CMP), lefthi, imm);
2600   } else {
2601     emit_mrm(as, XO_CMP, lefthi, righthi);
2602   }
2603 }
2604 #endif
2605 
2606 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
2607 
2608 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
2609 static void asm_hiop(ASMState *as, IRIns *ir)
2610 {
2611 #if LJ_32 && LJ_HASFFI
2612   /* HIOP is marked as a store because it needs its own DCE logic. */
2613   int uselo = ra_used(ir-1), usehi = ra_used(ir);  /* Loword/hiword used? */
2614   if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
2615   if ((ir-1)->o == IR_CONV) {  /* Conversions to/from 64 bit. */
2616     as->curins--;  /* Always skip the CONV. */
2617     if (usehi || uselo)
2618       asm_conv64(as, ir);
2619     return;
2620   } else if ((ir-1)->o <= IR_NE) {  /* 64 bit integer comparisons. ORDER IR. */
2621     asm_comp_int64(as, ir);
2622     return;
2623   } else if ((ir-1)->o == IR_XSTORE) {
2624     if ((ir-1)->r != RID_SINK)
2625       asm_fxstore(as, ir);
2626     return;
2627   }
2628   if (!usehi) return;  /* Skip unused hiword op for all remaining ops. */
2629   switch ((ir-1)->o) {
2630   case IR_ADD:
2631     as->flagmcp = NULL;
2632     as->curins--;
2633     asm_intarith(as, ir, XOg_ADC);
2634     asm_intarith(as, ir-1, XOg_ADD);
2635     break;
2636   case IR_SUB:
2637     as->flagmcp = NULL;
2638     as->curins--;
2639     asm_intarith(as, ir, XOg_SBB);
2640     asm_intarith(as, ir-1, XOg_SUB);
2641     break;
2642   case IR_NEG: {
2643     Reg dest = ra_dest(as, ir, RSET_GPR);
2644     emit_rr(as, XO_GROUP3, XOg_NEG, dest);
2645     emit_i8(as, 0);
2646     emit_rr(as, XO_ARITHi8, XOg_ADC, dest);
2647     ra_left(as, dest, ir->op1);
2648     as->curins--;
2649     asm_neg_not(as, ir-1, XOg_NEG);
2650     break;
2651     }
2652   case IR_CALLN:
2653   case IR_CALLXS:
2654     if (!uselo)
2655       ra_allocref(as, ir->op1, RID2RSET(RID_RETLO));  /* Mark lo op as used. */
2656     break;
2657   case IR_CNEWI:
2658     /* Nothing to do here. Handled by CNEWI itself. */
2659     break;
2660   default: lua_assert(0); break;
2661   }
2662 #else
2663   UNUSED(as); UNUSED(ir); lua_assert(0);  /* Unused on x64 or without FFI. */
2664 #endif
2665 }
2666 
2667 /* -- Profiling ----------------------------------------------------------- */
2668 
2669 static void asm_prof(ASMState *as, IRIns *ir)
2670 {
2671   UNUSED(ir);
2672   asm_guardcc(as, CC_NE);
2673   emit_i8(as, HOOK_PROFILE);
2674   emit_rma(as, XO_GROUP3b, XOg_TEST, &J2G(as->J)->hookmask);
2675 }
2676 
2677 /* -- Stack handling ------------------------------------------------------ */
2678 
2679 /* Check Lua stack size for overflow. Use exit handler as fallback. */
2680 static void asm_stack_check(ASMState *as, BCReg topslot,
2681                             IRIns *irp, RegSet allow, ExitNo exitno)
2682 {
2683   /* Try to get an unused temp. register, otherwise spill/restore eax. */
2684   Reg pbase = irp ? irp->r : RID_BASE;
2685   Reg r = allow ? rset_pickbot(allow) : RID_EAX;
2686   emit_jcc(as, CC_B, exitstub_addr(as->J, exitno));
2687   if (allow == RSET_EMPTY)  /* Restore temp. register. */
2688     emit_rmro(as, XO_MOV, r|REX_64, RID_ESP, 0);
2689   else
2690     ra_modified(as, r);
2691   emit_gri(as, XG_ARITHi(XOg_CMP), r|REX_GC64, (int32_t)(8*topslot));
2692   if (ra_hasreg(pbase) && pbase != r)
2693     emit_rr(as, XO_ARITH(XOg_SUB), r|REX_GC64, pbase);
2694   else
2695 #if LJ_GC64
2696     emit_rmro(as, XO_ARITH(XOg_SUB), r|REX_64, RID_DISPATCH,
2697               (int32_t)dispofs(as, &J2G(as->J)->jit_base));
2698 #else
2699     emit_rmro(as, XO_ARITH(XOg_SUB), r, RID_NONE,
2700               ptr2addr(&J2G(as->J)->jit_base));
2701 #endif
2702   emit_rmro(as, XO_MOV, r|REX_GC64, r, offsetof(lua_State, maxstack));
2703   emit_getgl(as, r, cur_L);
2704   if (allow == RSET_EMPTY)  /* Spill temp. register. */
2705     emit_rmro(as, XO_MOVto, r|REX_64, RID_ESP, 0);
2706 }
2707 
2708 /* Restore Lua stack from on-trace state. */
2709 static void asm_stack_restore(ASMState *as, SnapShot *snap)
2710 {
2711   SnapEntry *map = &as->T->snapmap[snap->mapofs];
2712 #if !LJ_FR2 || defined(LUA_USE_ASSERT)
2713   SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1-LJ_FR2];
2714 #endif
2715   MSize n, nent = snap->nent;
2716   /* Store the value of all modified slots to the Lua stack. */
2717   for (n = 0; n < nent; n++) {
2718     SnapEntry sn = map[n];
2719     BCReg s = snap_slot(sn);
2720     int32_t ofs = 8*((int32_t)s-1-LJ_FR2);
2721     IRRef ref = snap_ref(sn);
2722     IRIns *ir = IR(ref);
2723     if ((sn & SNAP_NORESTORE))
2724       continue;
2725     if (irt_isnum(ir->t)) {
2726       Reg src = ra_alloc1(as, ref, RSET_FPR);
2727       emit_rmro(as, XO_MOVSDto, src, RID_BASE, ofs);
2728     } else {
2729       lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) ||
2730                  (LJ_DUALNUM && irt_isinteger(ir->t)));
2731       if (!irref_isk(ref)) {
2732         Reg src = ra_alloc1(as, ref, rset_exclude(RSET_GPR, RID_BASE));
2733 #if LJ_GC64
2734         if (irt_is64(ir->t)) {
2735           /* TODO: 64 bit store + 32 bit load-modify-store is suboptimal. */
2736           emit_u32(as, irt_toitype(ir->t) << 15);
2737           emit_rmro(as, XO_ARITHi, XOg_OR, RID_BASE, ofs+4);
2738         } else if (LJ_DUALNUM && irt_isinteger(ir->t)) {
2739           emit_movmroi(as, RID_BASE, ofs+4, LJ_TISNUM << 15);
2740         } else {
2741           emit_movmroi(as, RID_BASE, ofs+4, (irt_toitype(ir->t)<<15)|0x7fff);
2742         }
2743 #endif
2744         emit_movtomro(as, REX_64IR(ir, src), RID_BASE, ofs);
2745 #if LJ_GC64
2746       } else {
2747         TValue k;
2748         lj_ir_kvalue(as->J->L, &k, ir);
2749         if (tvisnil(&k)) {
2750           emit_i32(as, -1);
2751           emit_rmro(as, XO_MOVmi, REX_64, RID_BASE, ofs);
2752         } else {
2753           emit_movmroi(as, RID_BASE, ofs+4, k.u32.hi);
2754           emit_movmroi(as, RID_BASE, ofs, k.u32.lo);
2755         }
2756 #else
2757       } else if (!irt_ispri(ir->t)) {
2758         emit_movmroi(as, RID_BASE, ofs, ir->i);
2759 #endif
2760       }
2761       if ((sn & (SNAP_CONT|SNAP_FRAME))) {
2762 #if !LJ_FR2
2763         if (s != 0)  /* Do not overwrite link to previous frame. */
2764           emit_movmroi(as, RID_BASE, ofs+4, (int32_t)(*flinks--));
2765 #endif
2766 #if !LJ_GC64
2767       } else {
2768         if (!(LJ_64 && irt_islightud(ir->t)))
2769           emit_movmroi(as, RID_BASE, ofs+4, irt_toitype(ir->t));
2770 #endif
2771       }
2772     }
2773     checkmclim(as);
2774   }
2775   lua_assert(map + nent == flinks);
2776 }
2777 
2778 /* -- GC handling --------------------------------------------------------- */
2779 
2780 /* Check GC threshold and do one or more GC steps. */
2781 static void asm_gc_check(ASMState *as)
2782 {
2783   const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_step_jit];
2784   IRRef args[2];
2785   MCLabel l_end;
2786   Reg tmp;
2787   ra_evictset(as, RSET_SCRATCH);
2788   l_end = emit_label(as);
2789   /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
2790   asm_guardcc(as, CC_NE);  /* Assumes asm_snap_prep() already done. */
2791   emit_rr(as, XO_TEST, RID_RET, RID_RET);
2792   args[0] = ASMREF_TMP1;  /* global_State *g */
2793   args[1] = ASMREF_TMP2;  /* MSize steps     */
2794   asm_gencall(as, ci, args);
2795   tmp = ra_releasetmp(as, ASMREF_TMP1);
2796 #if LJ_GC64
2797   emit_rmro(as, XO_LEA, tmp|REX_64, RID_DISPATCH, GG_DISP2G);
2798 #else
2799   emit_loada(as, tmp, J2G(as->J));
2800 #endif
2801   emit_loadi(as, ra_releasetmp(as, ASMREF_TMP2), as->gcsteps);
2802   /* Jump around GC step if GC total < GC threshold. */
2803   emit_sjcc(as, CC_B, l_end);
2804   emit_opgl(as, XO_ARITH(XOg_CMP), tmp|REX_GC64, gc.threshold);
2805   emit_getgl(as, tmp, gc.total);
2806   as->gcsteps = 0;
2807   checkmclim(as);
2808 }
2809 
2810 /* -- Loop handling ------------------------------------------------------- */
2811 
2812 /* Fixup the loop branch. */
2813 static void asm_loop_fixup(ASMState *as)
2814 {
2815   MCode *p = as->mctop;
2816   MCode *target = as->mcp;
2817   if (as->realign) {  /* Realigned loops use short jumps. */
2818     as->realign = NULL;  /* Stop another retry. */
2819     lua_assert(((intptr_t)target & 15) == 0);
2820     if (as->loopinv) {  /* Inverted loop branch? */
2821       p -= 5;
2822       p[0] = XI_JMP;
2823       lua_assert(target - p >= -128);
2824       p[-1] = (MCode)(target - p);  /* Patch sjcc. */
2825       if (as->loopinv == 2)
2826         p[-3] = (MCode)(target - p + 2);  /* Patch opt. short jp. */
2827     } else {
2828       lua_assert(target - p >= -128);
2829       p[-1] = (MCode)(int8_t)(target - p);  /* Patch short jmp. */
2830       p[-2] = XI_JMPs;
2831     }
2832   } else {
2833     MCode *newloop;
2834     p[-5] = XI_JMP;
2835     if (as->loopinv) {  /* Inverted loop branch? */
2836       /* asm_guardcc already inverted the jcc and patched the jmp. */
2837       p -= 5;
2838       newloop = target+4;
2839       *(int32_t *)(p-4) = (int32_t)(target - p);  /* Patch jcc. */
2840       if (as->loopinv == 2) {
2841         *(int32_t *)(p-10) = (int32_t)(target - p + 6);  /* Patch opt. jp. */
2842         newloop = target+8;
2843       }
2844     } else {  /* Otherwise just patch jmp. */
2845       *(int32_t *)(p-4) = (int32_t)(target - p);
2846       newloop = target+3;
2847     }
2848     /* Realign small loops and shorten the loop branch. */
2849     if (newloop >= p - 128) {
2850       as->realign = newloop;  /* Force a retry and remember alignment. */
2851       as->curins = as->stopins;  /* Abort asm_trace now. */
2852       as->T->nins = as->orignins;  /* Remove any added renames. */
2853     }
2854   }
2855 }
2856 
2857 /* -- Head of trace ------------------------------------------------------- */
2858 
2859 /* Coalesce BASE register for a root trace. */
2860 static void asm_head_root_base(ASMState *as)
2861 {
2862   IRIns *ir = IR(REF_BASE);
2863   Reg r = ir->r;
2864   if (ra_hasreg(r)) {
2865     ra_free(as, r);
2866     if (rset_test(as->modset, r) || irt_ismarked(ir->t))
2867       ir->r = RID_INIT;  /* No inheritance for modified BASE register. */
2868     if (r != RID_BASE)
2869       emit_rr(as, XO_MOV, r|REX_GC64, RID_BASE);
2870   }
2871 }
2872 
2873 /* Coalesce or reload BASE register for a side trace. */
2874 static RegSet asm_head_side_base(ASMState *as, IRIns *irp, RegSet allow)
2875 {
2876   IRIns *ir = IR(REF_BASE);
2877   Reg r = ir->r;
2878   if (ra_hasreg(r)) {
2879     ra_free(as, r);
2880     if (rset_test(as->modset, r) || irt_ismarked(ir->t))
2881       ir->r = RID_INIT;  /* No inheritance for modified BASE register. */
2882     if (irp->r == r) {
2883       rset_clear(allow, r);  /* Mark same BASE register as coalesced. */
2884     } else if (ra_hasreg(irp->r) && rset_test(as->freeset, irp->r)) {
2885       /* Move from coalesced parent reg. */
2886       rset_clear(allow, irp->r);
2887       emit_rr(as, XO_MOV, r|REX_GC64, irp->r);
2888     } else {
2889       emit_getgl(as, r, jit_base);  /* Otherwise reload BASE. */
2890     }
2891   }
2892   return allow;
2893 }
2894 
2895 /* -- Tail of trace ------------------------------------------------------- */
2896 
2897 /* Fixup the tail code. */
2898 static void asm_tail_fixup(ASMState *as, TraceNo lnk)
2899 {
2900   /* Note: don't use as->mcp swap + emit_*: emit_op overwrites more bytes. */
2901   MCode *p = as->mctop;
2902   MCode *target, *q;
2903   int32_t spadj = as->T->spadjust;
2904   if (spadj == 0) {
2905     p -= ((as->flags & JIT_F_LEA_AGU) ? 7 : 6) + (LJ_64 ? 1 : 0);
2906   } else {
2907     MCode *p1;
2908     /* Patch stack adjustment. */
2909     if (checki8(spadj)) {
2910       p -= 3;
2911       p1 = p-6;
2912       *p1 = (MCode)spadj;
2913     } else {
2914       p1 = p-9;
2915       *(int32_t *)p1 = spadj;
2916     }
2917     if ((as->flags & JIT_F_LEA_AGU)) {
2918 #if LJ_64
2919       p1[-4] = 0x48;
2920 #endif
2921       p1[-3] = (MCode)XI_LEA;
2922       p1[-2] = MODRM(checki8(spadj) ? XM_OFS8 : XM_OFS32, RID_ESP, RID_ESP);
2923       p1[-1] = MODRM(XM_SCALE1, RID_ESP, RID_ESP);
2924     } else {
2925 #if LJ_64
2926       p1[-3] = 0x48;
2927 #endif
2928       p1[-2] = (MCode)(checki8(spadj) ? XI_ARITHi8 : XI_ARITHi);
2929       p1[-1] = MODRM(XM_REG, XOg_ADD, RID_ESP);
2930     }
2931   }
2932   /* Patch exit branch. */
2933   target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp;
2934   *(int32_t *)(p-4) = jmprel(p, target);
2935   p[-5] = XI_JMP;
2936   /* Drop unused mcode tail. Fill with NOPs to make the prefetcher happy. */
2937   for (q = as->mctop-1; q >= p; q--)
2938     *q = XI_NOP;
2939   as->mctop = p;
2940 }
2941 
2942 /* Prepare tail of code. */
2943 static void asm_tail_prep(ASMState *as)
2944 {
2945   MCode *p = as->mctop;
2946   /* Realign and leave room for backwards loop branch or exit branch. */
2947   if (as->realign) {
2948     int i = ((int)(intptr_t)as->realign) & 15;
2949     /* Fill unused mcode tail with NOPs to make the prefetcher happy. */
2950     while (i-- > 0)
2951       *--p = XI_NOP;
2952     as->mctop = p;
2953     p -= (as->loopinv ? 5 : 2);  /* Space for short/near jmp. */
2954   } else {
2955     p -= 5;  /* Space for exit branch (near jmp). */
2956   }
2957   if (as->loopref) {
2958     as->invmcp = as->mcp = p;
2959   } else {
2960     /* Leave room for ESP adjustment: add esp, imm or lea esp, [esp+imm] */
2961     as->mcp = p - (((as->flags & JIT_F_LEA_AGU) ? 7 : 6)  + (LJ_64 ? 1 : 0));
2962     as->invmcp = NULL;
2963   }
2964 }
2965 
2966 /* -- Trace setup --------------------------------------------------------- */
2967 
2968 /* Ensure there are enough stack slots for call arguments. */
2969 static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
2970 {
2971   IRRef args[CCI_NARGS_MAX*2];
2972   int nslots;
2973   asm_collectargs(as, ir, ci, args);
2974   nslots = asm_count_call_slots(as, ci, args);
2975   if (nslots > as->evenspill)  /* Leave room for args in stack slots. */
2976     as->evenspill = nslots;
2977 #if LJ_64
2978   return irt_isfp(ir->t) ? REGSP_HINT(RID_FPRET) : REGSP_HINT(RID_RET);
2979 #else
2980   return irt_isfp(ir->t) ? REGSP_INIT : REGSP_HINT(RID_RET);
2981 #endif
2982 }
2983 
2984 /* Target-specific setup. */
2985 static void asm_setup_target(ASMState *as)
2986 {
2987   asm_exitstub_setup(as, as->T->nsnap);
2988   as->mrm.base = 0;
2989 }
2990 
2991 /* -- Trace patching ------------------------------------------------------ */
2992 
2993 static const uint8_t map_op1[256] = {
2994 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20,
2995 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,
2996 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,
2997 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,
2998 #if LJ_64
2999 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14,
3000 #else
3001 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,
3002 #endif
3003 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,
3004 0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51,
3005 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,
3006 0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,
3007 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51,
3008 #if LJ_64
3009 0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51,
3010 #else
3011 0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51,
3012 #endif
3013 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05,
3014 0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51,
3015 0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,
3016 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51,
3017 0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92
3018 };
3019 
3020 static const uint8_t map_op2[256] = {
3021 0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94,
3022 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
3023 0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
3024 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51,
3025 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
3026 0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
3027 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
3028 0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
3029 0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,
3030 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
3031 0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93,
3032 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93,
3033 0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,
3034 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
3035 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
3036 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52
3037 };
3038 
3039 static uint32_t asm_x86_inslen(const uint8_t* p)
3040 {
3041   uint32_t result = 0;
3042   uint32_t prefixes = 0;
3043   uint32_t x = map_op1[*p];
3044   for (;;) {
3045     switch (x >> 4) {
3046     case 0: return result + x + (prefixes & 4);
3047     case 1: prefixes |= x; x = map_op1[*++p]; result++; break;
3048     case 2: x = map_op2[*++p]; break;
3049     case 3: p++; goto mrm;
3050     case 4: result -= (prefixes & 2);  /* fallthrough */
3051     case 5: return result + (x & 15);
3052     case 6:  /* Group 3. */
3053       if (p[1] & 0x38) x = 2;
3054       else if ((prefixes & 2) && (x == 0x66)) x = 4;
3055       goto mrm;
3056     case 7: /* VEX c4/c5. */
3057       if (LJ_32 && p[1] < 0xc0) {
3058         x = 2;
3059         goto mrm;
3060       }
3061       if (x == 0x70) {
3062         x = *++p & 0x1f;
3063         result++;
3064         if (x >= 2) {
3065           p += 2;
3066           result += 2;
3067           goto mrm;
3068         }
3069       }
3070       p++;
3071       result++;
3072       x = map_op2[*++p];
3073       break;
3074     case 8: result -= (prefixes & 2);  /* fallthrough */
3075     case 9: mrm:  /* ModR/M and possibly SIB. */
3076       result += (x & 15);
3077       x = *++p;
3078       switch (x >> 6) {
3079       case 0: if ((x & 7) == 5) return result + 4; break;
3080       case 1: result++; break;
3081       case 2: result += 4; break;
3082       case 3: return result;
3083       }
3084       if ((x & 7) == 4) {
3085         result++;
3086         if (x < 0x40 && (p[1] & 7) == 5) result += 4;
3087       }
3088       return result;
3089     }
3090   }
3091 }
3092 
3093 /* Patch exit jumps of existing machine code to a new target. */
3094 void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
3095 {
3096   MCode *p = T->mcode;
3097   MCode *mcarea = lj_mcode_patch(J, p, 0);
3098   MSize len = T->szmcode;
3099   MCode *px = exitstub_addr(J, exitno) - 6;
3100   MCode *pe = p+len-6;
3101 #if LJ_GC64
3102   uint32_t statei = (uint32_t)(GG_OFS(g.vmstate) - GG_OFS(dispatch));
3103 #else
3104   uint32_t statei = u32ptr(&J2G(J)->vmstate);
3105 #endif
3106   if (len > 5 && p[len-5] == XI_JMP && p+len-6 + *(int32_t *)(p+len-4) == px)
3107     *(int32_t *)(p+len-4) = jmprel(p+len, target);
3108   /* Do not patch parent exit for a stack check. Skip beyond vmstate update. */
3109   for (; p < pe; p += asm_x86_inslen(p)) {
3110     intptr_t ofs = LJ_GC64 ? (p[0] & 0xf0) == 0x40 : LJ_64;
3111     if (*(uint32_t *)(p+2+ofs) == statei && p[ofs+LJ_GC64-LJ_64] == XI_MOVmi)
3112       break;
3113   }
3114   lua_assert(p < pe);
3115   for (; p < pe; p += asm_x86_inslen(p))
3116     if ((*(uint16_t *)p & 0xf0ff) == 0x800f && p + *(int32_t *)(p+2) == px)
3117       *(int32_t *)(p+2) = jmprel(p+6, target);
3118   lj_mcode_sync(T->mcode, T->mcode + T->szmcode);
3119   lj_mcode_patch(J, mcarea, 1);
3120 }
3121 

/* [<][>][^][v][top][bottom][index][help] */