root/lj_emit_x86.h

/* [<][>][^][v][top][bottom][index][help] */

INCLUDED FROM


DEFINITIONS

This source file includes following definitions.
  1. emit_op
  2. emit_rr
  3. ptr2addr
  4. emit_rma
  5. emit_rmro
  6. emit_rmrxo
  7. emit_gri
  8. emit_gmroi
  9. emit_mrm
  10. emit_gmrmi
  11. emit_movmroi
  12. emit_loadi
  13. emit_loadu64
  14. emit_loadn
  15. emit_sjmp
  16. emit_sjcc
  17. emit_sjcc_label
  18. emit_sfixup
  19. jmprel
  20. emit_jcc
  21. emit_jmp
  22. emit_call_
  23. emit_movrr
  24. emit_spload
  25. emit_spstore
  26. emit_addptr

   1 /*
   2 ** x86/x64 instruction emitter.
   3 ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5 
   6 /* -- Emit basic instructions --------------------------------------------- */
   7 
   8 #define MODRM(mode, r1, r2)     ((MCode)((mode)+(((r1)&7)<<3)+((r2)&7)))
   9 
  10 #if LJ_64
  11 #define REXRB(p, rr, rb) \
  12     { MCode rex = 0x40 + (((rr)>>1)&4) + (((rb)>>3)&1); \
  13       if (rex != 0x40) *--(p) = rex; }
  14 #define FORCE_REX               0x200
  15 #define REX_64                  (FORCE_REX|0x080000)
  16 #else
  17 #define REXRB(p, rr, rb)        ((void)0)
  18 #define FORCE_REX               0
  19 #define REX_64                  0
  20 #endif
  21 
  22 #define emit_i8(as, i)          (*--as->mcp = (MCode)(i))
  23 #define emit_i32(as, i)         (*(int32_t *)(as->mcp-4) = (i), as->mcp -= 4)
  24 #define emit_u32(as, u)         (*(uint32_t *)(as->mcp-4) = (u), as->mcp -= 4)
  25 
  26 #define emit_x87op(as, xo) \
  27   (*(uint16_t *)(as->mcp-2) = (uint16_t)(xo), as->mcp -= 2)
  28 
  29 /* op */
  30 static LJ_AINLINE MCode *emit_op(x86Op xo, Reg rr, Reg rb, Reg rx,
  31                                  MCode *p, int delta)
  32 {
  33   int n = (int8_t)xo;
  34 #if defined(__GNUC__)
  35   if (__builtin_constant_p(xo) && n == -2)
  36     p[delta-2] = (MCode)(xo >> 24);
  37   else if (__builtin_constant_p(xo) && n == -3)
  38     *(uint16_t *)(p+delta-3) = (uint16_t)(xo >> 16);
  39   else
  40 #endif
  41     *(uint32_t *)(p+delta-5) = (uint32_t)xo;
  42   p += n + delta;
  43 #if LJ_64
  44   {
  45     uint32_t rex = 0x40 + ((rr>>1)&(4+(FORCE_REX>>1)))+((rx>>2)&2)+((rb>>3)&1);
  46     if (rex != 0x40) {
  47       rex |= (rr >> 16);
  48       if (n == -4) { *p = (MCode)rex; rex = (MCode)(xo >> 8); }
  49       else if ((xo & 0xffffff) == 0x6600fd) { *p = (MCode)rex; rex = 0x66; }
  50       *--p = (MCode)rex;
  51     }
  52   }
  53 #else
  54   UNUSED(rr); UNUSED(rb); UNUSED(rx);
  55 #endif
  56   return p;
  57 }
  58 
  59 /* op + modrm */
  60 #define emit_opm(xo, mode, rr, rb, p, delta) \
  61   (p[(delta)-1] = MODRM((mode), (rr), (rb)), \
  62    emit_op((xo), (rr), (rb), 0, (p), (delta)))
  63 
  64 /* op + modrm + sib */
  65 #define emit_opmx(xo, mode, scale, rr, rb, rx, p) \
  66   (p[-1] = MODRM((scale), (rx), (rb)), \
  67    p[-2] = MODRM((mode), (rr), RID_ESP), \
  68    emit_op((xo), (rr), (rb), (rx), (p), -1))
  69 
  70 /* op r1, r2 */
  71 static void emit_rr(ASMState *as, x86Op xo, Reg r1, Reg r2)
  72 {
  73   MCode *p = as->mcp;
  74   as->mcp = emit_opm(xo, XM_REG, r1, r2, p, 0);
  75 }
  76 
  77 #if LJ_64 && defined(LUA_USE_ASSERT)
  78 /* [addr] is sign-extended in x64 and must be in lower 2G (not 4G). */
  79 static int32_t ptr2addr(const void *p)
  80 {
  81   lua_assert((uintptr_t)p < (uintptr_t)0x80000000);
  82   return i32ptr(p);
  83 }
  84 #else
  85 #define ptr2addr(p)     (i32ptr((p)))
  86 #endif
  87 
  88 /* op r, [addr] */
  89 static void emit_rma(ASMState *as, x86Op xo, Reg rr, const void *addr)
  90 {
  91   MCode *p = as->mcp;
  92   *(int32_t *)(p-4) = ptr2addr(addr);
  93 #if LJ_64
  94   p[-5] = MODRM(XM_SCALE1, RID_ESP, RID_EBP);
  95   as->mcp = emit_opm(xo, XM_OFS0, rr, RID_ESP, p, -5);
  96 #else
  97   as->mcp = emit_opm(xo, XM_OFS0, rr, RID_EBP, p, -4);
  98 #endif
  99 }
 100 
 101 /* op r, [base+ofs] */
 102 static void emit_rmro(ASMState *as, x86Op xo, Reg rr, Reg rb, int32_t ofs)
 103 {
 104   MCode *p = as->mcp;
 105   x86Mode mode;
 106   if (ra_hasreg(rb)) {
 107     if (ofs == 0 && (rb&7) != RID_EBP) {
 108       mode = XM_OFS0;
 109     } else if (checki8(ofs)) {
 110       *--p = (MCode)ofs;
 111       mode = XM_OFS8;
 112     } else {
 113       p -= 4;
 114       *(int32_t *)p = ofs;
 115       mode = XM_OFS32;
 116     }
 117     if ((rb&7) == RID_ESP)
 118       *--p = MODRM(XM_SCALE1, RID_ESP, RID_ESP);
 119   } else {
 120     *(int32_t *)(p-4) = ofs;
 121 #if LJ_64
 122     p[-5] = MODRM(XM_SCALE1, RID_ESP, RID_EBP);
 123     p -= 5;
 124     rb = RID_ESP;
 125 #else
 126     p -= 4;
 127     rb = RID_EBP;
 128 #endif
 129     mode = XM_OFS0;
 130   }
 131   as->mcp = emit_opm(xo, mode, rr, rb, p, 0);
 132 }
 133 
 134 /* op r, [base+idx*scale+ofs] */
 135 static void emit_rmrxo(ASMState *as, x86Op xo, Reg rr, Reg rb, Reg rx,
 136                        x86Mode scale, int32_t ofs)
 137 {
 138   MCode *p = as->mcp;
 139   x86Mode mode;
 140   if (ofs == 0 && (rb&7) != RID_EBP) {
 141     mode = XM_OFS0;
 142   } else if (checki8(ofs)) {
 143     mode = XM_OFS8;
 144     *--p = (MCode)ofs;
 145   } else {
 146     mode = XM_OFS32;
 147     p -= 4;
 148     *(int32_t *)p = ofs;
 149   }
 150   as->mcp = emit_opmx(xo, mode, scale, rr, rb, rx, p);
 151 }
 152 
 153 /* op r, i */
 154 static void emit_gri(ASMState *as, x86Group xg, Reg rb, int32_t i)
 155 {
 156   MCode *p = as->mcp;
 157   x86Op xo;
 158   if (checki8(i)) {
 159     *--p = (MCode)i;
 160     xo = XG_TOXOi8(xg);
 161   } else {
 162     p -= 4;
 163     *(int32_t *)p = i;
 164     xo = XG_TOXOi(xg);
 165   }
 166   as->mcp = emit_opm(xo, XM_REG, (Reg)(xg & 7) | (rb & REX_64), rb, p, 0);
 167 }
 168 
 169 /* op [base+ofs], i */
 170 static void emit_gmroi(ASMState *as, x86Group xg, Reg rb, int32_t ofs,
 171                        int32_t i)
 172 {
 173   x86Op xo;
 174   if (checki8(i)) {
 175     emit_i8(as, i);
 176     xo = XG_TOXOi8(xg);
 177   } else {
 178     emit_i32(as, i);
 179     xo = XG_TOXOi(xg);
 180   }
 181   emit_rmro(as, xo, (Reg)(xg & 7), rb, ofs);
 182 }
 183 
 184 #define emit_shifti(as, xg, r, i) \
 185   (emit_i8(as, (i)), emit_rr(as, XO_SHIFTi, (Reg)(xg), (r)))
 186 
 187 /* op r, rm/mrm */
 188 static void emit_mrm(ASMState *as, x86Op xo, Reg rr, Reg rb)
 189 {
 190   MCode *p = as->mcp;
 191   x86Mode mode = XM_REG;
 192   if (rb == RID_MRM) {
 193     rb = as->mrm.base;
 194     if (rb == RID_NONE) {
 195       rb = RID_EBP;
 196       mode = XM_OFS0;
 197       p -= 4;
 198       *(int32_t *)p = as->mrm.ofs;
 199       if (as->mrm.idx != RID_NONE)
 200         goto mrmidx;
 201 #if LJ_64
 202       *--p = MODRM(XM_SCALE1, RID_ESP, RID_EBP);
 203       rb = RID_ESP;
 204 #endif
 205     } else {
 206       if (as->mrm.ofs == 0 && (rb&7) != RID_EBP) {
 207         mode = XM_OFS0;
 208       } else if (checki8(as->mrm.ofs)) {
 209         *--p = (MCode)as->mrm.ofs;
 210         mode = XM_OFS8;
 211       } else {
 212         p -= 4;
 213         *(int32_t *)p = as->mrm.ofs;
 214         mode = XM_OFS32;
 215       }
 216       if (as->mrm.idx != RID_NONE) {
 217       mrmidx:
 218         as->mcp = emit_opmx(xo, mode, as->mrm.scale, rr, rb, as->mrm.idx, p);
 219         return;
 220       }
 221       if ((rb&7) == RID_ESP)
 222         *--p = MODRM(XM_SCALE1, RID_ESP, RID_ESP);
 223     }
 224   }
 225   as->mcp = emit_opm(xo, mode, rr, rb, p, 0);
 226 }
 227 
 228 /* op rm/mrm, i */
 229 static void emit_gmrmi(ASMState *as, x86Group xg, Reg rb, int32_t i)
 230 {
 231   x86Op xo;
 232   if (checki8(i)) {
 233     emit_i8(as, i);
 234     xo = XG_TOXOi8(xg);
 235   } else {
 236     emit_i32(as, i);
 237     xo = XG_TOXOi(xg);
 238   }
 239   emit_mrm(as, xo, (Reg)(xg & 7) | (rb & REX_64), (rb & ~REX_64));
 240 }
 241 
 242 /* -- Emit loads/stores --------------------------------------------------- */
 243 
 244 /* Instruction selection for XMM moves. */
 245 #define XMM_MOVRR(as)   ((as->flags & JIT_F_SPLIT_XMM) ? XO_MOVSD : XO_MOVAPS)
 246 #define XMM_MOVRM(as)   ((as->flags & JIT_F_SPLIT_XMM) ? XO_MOVLPD : XO_MOVSD)
 247 
 248 /* mov [base+ofs], i */
 249 static void emit_movmroi(ASMState *as, Reg base, int32_t ofs, int32_t i)
 250 {
 251   emit_i32(as, i);
 252   emit_rmro(as, XO_MOVmi, 0, base, ofs);
 253 }
 254 
 255 /* mov [base+ofs], r */
 256 #define emit_movtomro(as, r, base, ofs) \
 257   emit_rmro(as, XO_MOVto, (r), (base), (ofs))
 258 
 259 /* Get/set global_State fields. */
 260 #define emit_opgl(as, xo, r, field) \
 261   emit_rma(as, (xo), (r), (void *)&J2G(as->J)->field)
 262 #define emit_getgl(as, r, field)        emit_opgl(as, XO_MOV, (r), field)
 263 #define emit_setgl(as, r, field)        emit_opgl(as, XO_MOVto, (r), field)
 264 
 265 #define emit_setvmstate(as, i) \
 266   (emit_i32(as, i), emit_opgl(as, XO_MOVmi, 0, vmstate))
 267 
 268 /* mov r, i / xor r, r */
 269 static void emit_loadi(ASMState *as, Reg r, int32_t i)
 270 {
 271   /* XOR r,r is shorter, but modifies the flags. This is bad for HIOP/jcc. */
 272   if (i == 0 && !(LJ_32 && (IR(as->curins)->o == IR_HIOP ||
 273                             (as->curins+1 < as->T->nins &&
 274                              IR(as->curins+1)->o == IR_HIOP))) &&
 275                 !((*as->mcp == 0x0f && (as->mcp[1] & 0xf0) == XI_JCCn) ||
 276                   (*as->mcp & 0xf0) == XI_JCCs)) {
 277     emit_rr(as, XO_ARITH(XOg_XOR), r, r);
 278   } else {
 279     MCode *p = as->mcp;
 280     *(int32_t *)(p-4) = i;
 281     p[-5] = (MCode)(XI_MOVri+(r&7));
 282     p -= 5;
 283     REXRB(p, 0, r);
 284     as->mcp = p;
 285   }
 286 }
 287 
 288 /* mov r, addr */
 289 #define emit_loada(as, r, addr) \
 290   emit_loadi(as, (r), ptr2addr((addr)))
 291 
 292 #if LJ_64
 293 /* mov r, imm64 or shorter 32 bit extended load. */
 294 static void emit_loadu64(ASMState *as, Reg r, uint64_t u64)
 295 {
 296   if (checku32(u64)) {  /* 32 bit load clears upper 32 bits. */
 297     emit_loadi(as, r, (int32_t)u64);
 298   } else if (checki32((int64_t)u64)) {  /* Sign-extended 32 bit load. */
 299     MCode *p = as->mcp;
 300     *(int32_t *)(p-4) = (int32_t)u64;
 301     as->mcp = emit_opm(XO_MOVmi, XM_REG, REX_64, r, p, -4);
 302   } else {  /* Full-size 64 bit load. */
 303     MCode *p = as->mcp;
 304     *(uint64_t *)(p-8) = u64;
 305     p[-9] = (MCode)(XI_MOVri+(r&7));
 306     p[-10] = 0x48 + ((r>>3)&1);
 307     p -= 10;
 308     as->mcp = p;
 309   }
 310 }
 311 #endif
 312 
 313 /* movsd r, [&tv->n] / xorps r, r */
 314 static void emit_loadn(ASMState *as, Reg r, cTValue *tv)
 315 {
 316   if (tvispzero(tv))  /* Use xor only for +0. */
 317     emit_rr(as, XO_XORPS, r, r);
 318   else
 319     emit_rma(as, XMM_MOVRM(as), r, &tv->n);
 320 }
 321 
 322 /* -- Emit control-flow instructions -------------------------------------- */
 323 
 324 /* Label for short jumps. */
 325 typedef MCode *MCLabel;
 326 
 327 #if LJ_32 && LJ_HASFFI
 328 /* jmp short target */
 329 static void emit_sjmp(ASMState *as, MCLabel target)
 330 {
 331   MCode *p = as->mcp;
 332   ptrdiff_t delta = target - p;
 333   lua_assert(delta == (int8_t)delta);
 334   p[-1] = (MCode)(int8_t)delta;
 335   p[-2] = XI_JMPs;
 336   as->mcp = p - 2;
 337 }
 338 #endif
 339 
 340 /* jcc short target */
 341 static void emit_sjcc(ASMState *as, int cc, MCLabel target)
 342 {
 343   MCode *p = as->mcp;
 344   ptrdiff_t delta = target - p;
 345   lua_assert(delta == (int8_t)delta);
 346   p[-1] = (MCode)(int8_t)delta;
 347   p[-2] = (MCode)(XI_JCCs+(cc&15));
 348   as->mcp = p - 2;
 349 }
 350 
 351 /* jcc short (pending target) */
 352 static MCLabel emit_sjcc_label(ASMState *as, int cc)
 353 {
 354   MCode *p = as->mcp;
 355   p[-1] = 0;
 356   p[-2] = (MCode)(XI_JCCs+(cc&15));
 357   as->mcp = p - 2;
 358   return p;
 359 }
 360 
 361 /* Fixup jcc short target. */
 362 static void emit_sfixup(ASMState *as, MCLabel source)
 363 {
 364   source[-1] = (MCode)(as->mcp-source);
 365 }
 366 
 367 /* Return label pointing to current PC. */
 368 #define emit_label(as)          ((as)->mcp)
 369 
 370 /* Compute relative 32 bit offset for jump and call instructions. */
 371 static LJ_AINLINE int32_t jmprel(MCode *p, MCode *target)
 372 {
 373   ptrdiff_t delta = target - p;
 374   lua_assert(delta == (int32_t)delta);
 375   return (int32_t)delta;
 376 }
 377 
 378 /* jcc target */
 379 static void emit_jcc(ASMState *as, int cc, MCode *target)
 380 {
 381   MCode *p = as->mcp;
 382   *(int32_t *)(p-4) = jmprel(p, target);
 383   p[-5] = (MCode)(XI_JCCn+(cc&15));
 384   p[-6] = 0x0f;
 385   as->mcp = p - 6;
 386 }
 387 
 388 /* jmp target */
 389 static void emit_jmp(ASMState *as, MCode *target)
 390 {
 391   MCode *p = as->mcp;
 392   *(int32_t *)(p-4) = jmprel(p, target);
 393   p[-5] = XI_JMP;
 394   as->mcp = p - 5;
 395 }
 396 
 397 /* call target */
 398 static void emit_call_(ASMState *as, MCode *target)
 399 {
 400   MCode *p = as->mcp;
 401 #if LJ_64
 402   if (target-p != (int32_t)(target-p)) {
 403     /* Assumes RID_RET is never an argument to calls and always clobbered. */
 404     emit_rr(as, XO_GROUP5, XOg_CALL, RID_RET);
 405     emit_loadu64(as, RID_RET, (uint64_t)target);
 406     return;
 407   }
 408 #endif
 409   *(int32_t *)(p-4) = jmprel(p, target);
 410   p[-5] = XI_CALL;
 411   as->mcp = p - 5;
 412 }
 413 
 414 #define emit_call(as, f)        emit_call_(as, (MCode *)(void *)(f))
 415 
 416 /* -- Emit generic operations --------------------------------------------- */
 417 
 418 /* Use 64 bit operations to handle 64 bit IR types. */
 419 #if LJ_64
 420 #define REX_64IR(ir, r)         ((r) + (irt_is64((ir)->t) ? REX_64 : 0))
 421 #else
 422 #define REX_64IR(ir, r)         (r)
 423 #endif
 424 
 425 /* Generic move between two regs. */
 426 static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src)
 427 {
 428   UNUSED(ir);
 429   if (dst < RID_MAX_GPR)
 430     emit_rr(as, XO_MOV, REX_64IR(ir, dst), src);
 431   else
 432     emit_rr(as, XMM_MOVRR(as), dst, src);
 433 }
 434 
 435 /* Generic load of register from stack slot. */
 436 static void emit_spload(ASMState *as, IRIns *ir, Reg r, int32_t ofs)
 437 {
 438   if (r < RID_MAX_GPR)
 439     emit_rmro(as, XO_MOV, REX_64IR(ir, r), RID_ESP, ofs);
 440   else
 441     emit_rmro(as, irt_isnum(ir->t) ? XMM_MOVRM(as) : XO_MOVSS, r, RID_ESP, ofs);
 442 }
 443 
 444 /* Generic store of register to stack slot. */
 445 static void emit_spstore(ASMState *as, IRIns *ir, Reg r, int32_t ofs)
 446 {
 447   if (r < RID_MAX_GPR)
 448     emit_rmro(as, XO_MOVto, REX_64IR(ir, r), RID_ESP, ofs);
 449   else
 450     emit_rmro(as, irt_isnum(ir->t) ? XO_MOVSDto : XO_MOVSSto, r, RID_ESP, ofs);
 451 }
 452 
 453 /* Add offset to pointer. */
 454 static void emit_addptr(ASMState *as, Reg r, int32_t ofs)
 455 {
 456   if (ofs) {
 457     if ((as->flags & JIT_F_LEA_AGU))
 458       emit_rmro(as, XO_LEA, r, r, ofs);
 459     else
 460       emit_gri(as, XG_ARITHi(XOg_ADD), r, ofs);
 461   }
 462 }
 463 
 464 #define emit_spsub(as, ofs)     emit_addptr(as, RID_ESP|REX_64, -(ofs))
 465 
 466 /* Prefer rematerialization of BASE/L from global_State over spills. */
 467 #define emit_canremat(ref)      ((ref) <= REF_BASE)
 468 

/* [<][>][^][v][top][bottom][index][help] */