root/lj_record.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. rec_check_ir
  2. rec_check_slots
  3. sloadt
  4. sload
  5. getcurrf
  6. lj_record_objcmp
  7. lj_record_constify
  8. canonicalize_slots
  9. lj_record_stop
  10. find_kinit
  11. fori_load
  12. fori_arg
  13. rec_for_direction
  14. rec_for_iter
  15. rec_for_check
  16. rec_for_loop
  17. rec_for
  18. rec_iterl
  19. rec_loop
  20. innerloopleft
  21. rec_loop_interp
  22. rec_loop_jit
  23. rec_profile_need
  24. rec_profile_ins
  25. rec_profile_ret
  26. rec_call_specialize
  27. rec_call_setup
  28. lj_record_call
  29. lj_record_tailcall
  30. check_downrec_unroll
  31. lj_record_ret
  32. rec_mm_prep
  33. lj_record_mm_lookup
  34. rec_mm_arith
  35. rec_mm_len
  36. rec_mm_callcomp
  37. rec_mm_equal
  38. rec_mm_comp
  39. rec_mm_comp_cdata
  40. rec_idx_bump
  41. rec_idx_abc
  42. rec_idx_key
  43. nommstr
  44. lj_record_idx
  45. rec_tsetm
  46. rec_upvalue_constify
  47. rec_upvalue
  48. check_call_unroll
  49. rec_func_setup
  50. rec_func_vararg
  51. rec_func_lua
  52. rec_func_jit
  53. select_detect
  54. rec_varg
  55. rec_tnew
  56. rec_cat
  57. rec_comp_prep
  58. rec_comp_fixup
  59. lj_record_ins
  60. rec_setup_root
  61. lj_record_setup

   1 /*
   2 ** Trace recorder (bytecode -> SSA IR).
   3 ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5 
   6 #define lj_record_c
   7 #define LUA_CORE
   8 
   9 #include "lj_obj.h"
  10 
  11 #if LJ_HASJIT
  12 
  13 #include "lj_err.h"
  14 #include "lj_str.h"
  15 #include "lj_tab.h"
  16 #include "lj_meta.h"
  17 #include "lj_frame.h"
  18 #if LJ_HASFFI
  19 #include "lj_ctype.h"
  20 #endif
  21 #include "lj_bc.h"
  22 #include "lj_ff.h"
  23 #if LJ_HASPROFILE
  24 #include "lj_debug.h"
  25 #endif
  26 #include "lj_ir.h"
  27 #include "lj_jit.h"
  28 #include "lj_ircall.h"
  29 #include "lj_iropt.h"
  30 #include "lj_trace.h"
  31 #include "lj_record.h"
  32 #include "lj_ffrecord.h"
  33 #include "lj_snap.h"
  34 #include "lj_dispatch.h"
  35 #include "lj_vm.h"
  36 
  37 /* Some local macros to save typing. Undef'd at the end. */
  38 #define IR(ref)                 (&J->cur.ir[(ref)])
  39 
  40 /* Pass IR on to next optimization in chain (FOLD). */
  41 #define emitir(ot, a, b)        (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
  42 
  43 /* Emit raw IR without passing through optimizations. */
  44 #define emitir_raw(ot, a, b)    (lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J))
  45 
  46 /* -- Sanity checks ------------------------------------------------------- */
  47 
  48 #ifdef LUA_USE_ASSERT
  49 /* Sanity check the whole IR -- sloooow. */
  50 static void rec_check_ir(jit_State *J)
  51 {
  52   IRRef i, nins = J->cur.nins, nk = J->cur.nk;
  53   lua_assert(nk <= REF_BIAS && nins >= REF_BIAS && nins < 65536);
  54   for (i = nk; i < nins; i++) {
  55     IRIns *ir = IR(i);
  56     uint32_t mode = lj_ir_mode[ir->o];
  57     IRRef op1 = ir->op1;
  58     IRRef op2 = ir->op2;
  59     switch (irm_op1(mode)) {
  60     case IRMnone: lua_assert(op1 == 0); break;
  61     case IRMref: lua_assert(op1 >= nk);
  62       lua_assert(i >= REF_BIAS ? op1 < i : op1 > i); break;
  63     case IRMlit: break;
  64     case IRMcst: lua_assert(i < REF_BIAS);
  65       if (irt_is64(ir->t) && ir->o != IR_KNULL)
  66         i++;
  67       continue;
  68     }
  69     switch (irm_op2(mode)) {
  70     case IRMnone: lua_assert(op2 == 0); break;
  71     case IRMref: lua_assert(op2 >= nk);
  72       lua_assert(i >= REF_BIAS ? op2 < i : op2 > i); break;
  73     case IRMlit: break;
  74     case IRMcst: lua_assert(0); break;
  75     }
  76     if (ir->prev) {
  77       lua_assert(ir->prev >= nk);
  78       lua_assert(i >= REF_BIAS ? ir->prev < i : ir->prev > i);
  79       lua_assert(ir->o == IR_NOP || IR(ir->prev)->o == ir->o);
  80     }
  81   }
  82 }
  83 
  84 /* Compare stack slots and frames of the recorder and the VM. */
  85 static void rec_check_slots(jit_State *J)
  86 {
  87   BCReg s, nslots = J->baseslot + J->maxslot;
  88   int32_t depth = 0;
  89   cTValue *base = J->L->base - J->baseslot;
  90   lua_assert(J->baseslot >= 1+LJ_FR2);
  91   lua_assert(J->baseslot == 1+LJ_FR2 || (J->slot[J->baseslot-1] & TREF_FRAME));
  92   lua_assert(nslots <= LJ_MAX_JSLOTS);
  93   for (s = 0; s < nslots; s++) {
  94     TRef tr = J->slot[s];
  95     if (tr) {
  96       cTValue *tv = &base[s];
  97       IRRef ref = tref_ref(tr);
  98       IRIns *ir = NULL;  /* Silence compiler. */
  99       if (!LJ_FR2 || ref || !(tr & (TREF_FRAME | TREF_CONT))) {
 100         lua_assert(ref >= J->cur.nk && ref < J->cur.nins);
 101         ir = IR(ref);
 102         lua_assert(irt_t(ir->t) == tref_t(tr));
 103       }
 104       if (s == 0) {
 105         lua_assert(tref_isfunc(tr));
 106 #if LJ_FR2
 107       } else if (s == 1) {
 108         lua_assert((tr & ~TREF_FRAME) == 0);
 109 #endif
 110       } else if ((tr & TREF_FRAME)) {
 111         GCfunc *fn = gco2func(frame_gc(tv));
 112         BCReg delta = (BCReg)(tv - frame_prev(tv));
 113 #if LJ_FR2
 114         if (ref)
 115           lua_assert(ir_knum(ir)->u64 == tv->u64);
 116         tr = J->slot[s-1];
 117         ir = IR(tref_ref(tr));
 118 #endif
 119         lua_assert(tref_isfunc(tr));
 120         if (tref_isk(tr)) lua_assert(fn == ir_kfunc(ir));
 121         lua_assert(s > delta + LJ_FR2 ? (J->slot[s-delta] & TREF_FRAME)
 122                                       : (s == delta + LJ_FR2));
 123         depth++;
 124       } else if ((tr & TREF_CONT)) {
 125 #if LJ_FR2
 126         if (ref)
 127           lua_assert(ir_knum(ir)->u64 == tv->u64);
 128 #else
 129         lua_assert(ir_kptr(ir) == gcrefp(tv->gcr, void));
 130 #endif
 131         lua_assert((J->slot[s+1+LJ_FR2] & TREF_FRAME));
 132         depth++;
 133       } else {
 134         if (tvisnumber(tv))
 135           lua_assert(tref_isnumber(tr));  /* Could be IRT_INT etc., too. */
 136         else
 137           lua_assert(itype2irt(tv) == tref_type(tr));
 138         if (tref_isk(tr)) {  /* Compare constants. */
 139           TValue tvk;
 140           lj_ir_kvalue(J->L, &tvk, ir);
 141           if (!(tvisnum(&tvk) && tvisnan(&tvk)))
 142             lua_assert(lj_obj_equal(tv, &tvk));
 143           else
 144             lua_assert(tvisnum(tv) && tvisnan(tv));
 145         }
 146       }
 147     }
 148   }
 149   lua_assert(J->framedepth == depth);
 150 }
 151 #endif
 152 
 153 /* -- Type handling and specialization ------------------------------------ */
 154 
 155 /* Note: these functions return tagged references (TRef). */
 156 
 157 /* Specialize a slot to a specific type. Note: slot can be negative! */
 158 static TRef sloadt(jit_State *J, int32_t slot, IRType t, int mode)
 159 {
 160   /* Caller may set IRT_GUARD in t. */
 161   TRef ref = emitir_raw(IRT(IR_SLOAD, t), (int32_t)J->baseslot+slot, mode);
 162   J->base[slot] = ref;
 163   return ref;
 164 }
 165 
 166 /* Specialize a slot to the runtime type. Note: slot can be negative! */
 167 static TRef sload(jit_State *J, int32_t slot)
 168 {
 169   IRType t = itype2irt(&J->L->base[slot]);
 170   TRef ref = emitir_raw(IRTG(IR_SLOAD, t), (int32_t)J->baseslot+slot,
 171                         IRSLOAD_TYPECHECK);
 172   if (irtype_ispri(t)) ref = TREF_PRI(t);  /* Canonicalize primitive refs. */
 173   J->base[slot] = ref;
 174   return ref;
 175 }
 176 
 177 /* Get TRef from slot. Load slot and specialize if not done already. */
 178 #define getslot(J, s)   (J->base[(s)] ? J->base[(s)] : sload(J, (int32_t)(s)))
 179 
 180 /* Get TRef for current function. */
 181 static TRef getcurrf(jit_State *J)
 182 {
 183   if (J->base[-1-LJ_FR2])
 184     return J->base[-1-LJ_FR2];
 185   lua_assert(J->baseslot == 1+LJ_FR2);
 186   return sloadt(J, -1-LJ_FR2, IRT_FUNC, IRSLOAD_READONLY);
 187 }
 188 
 189 /* Compare for raw object equality.
 190 ** Returns 0 if the objects are the same.
 191 ** Returns 1 if they are different, but the same type.
 192 ** Returns 2 for two different types.
 193 ** Comparisons between primitives always return 1 -- no caller cares about it.
 194 */
 195 int lj_record_objcmp(jit_State *J, TRef a, TRef b, cTValue *av, cTValue *bv)
 196 {
 197   int diff = !lj_obj_equal(av, bv);
 198   if (!tref_isk2(a, b)) {  /* Shortcut, also handles primitives. */
 199     IRType ta = tref_isinteger(a) ? IRT_INT : tref_type(a);
 200     IRType tb = tref_isinteger(b) ? IRT_INT : tref_type(b);
 201     if (ta != tb) {
 202       /* Widen mixed number/int comparisons to number/number comparison. */
 203       if (ta == IRT_INT && tb == IRT_NUM) {
 204         a = emitir(IRTN(IR_CONV), a, IRCONV_NUM_INT);
 205         ta = IRT_NUM;
 206       } else if (ta == IRT_NUM && tb == IRT_INT) {
 207         b = emitir(IRTN(IR_CONV), b, IRCONV_NUM_INT);
 208       } else {
 209         return 2;  /* Two different types are never equal. */
 210       }
 211     }
 212     emitir(IRTG(diff ? IR_NE : IR_EQ, ta), a, b);
 213   }
 214   return diff;
 215 }
 216 
 217 /* Constify a value. Returns 0 for non-representable object types. */
 218 TRef lj_record_constify(jit_State *J, cTValue *o)
 219 {
 220   if (tvisgcv(o))
 221     return lj_ir_kgc(J, gcV(o), itype2irt(o));
 222   else if (tvisint(o))
 223     return lj_ir_kint(J, intV(o));
 224   else if (tvisnum(o))
 225     return lj_ir_knumint(J, numV(o));
 226   else if (tvisbool(o))
 227     return TREF_PRI(itype2irt(o));
 228   else
 229     return 0;  /* Can't represent lightuserdata (pointless). */
 230 }
 231 
 232 /* -- Record loop ops ----------------------------------------------------- */
 233 
 234 /* Loop event. */
 235 typedef enum {
 236   LOOPEV_LEAVE,         /* Loop is left or not entered. */
 237   LOOPEV_ENTERLO,       /* Loop is entered with a low iteration count left. */
 238   LOOPEV_ENTER          /* Loop is entered. */
 239 } LoopEvent;
 240 
 241 /* Canonicalize slots: convert integers to numbers. */
 242 static void canonicalize_slots(jit_State *J)
 243 {
 244   BCReg s;
 245   if (LJ_DUALNUM) return;
 246   for (s = J->baseslot+J->maxslot-1; s >= 1; s--) {
 247     TRef tr = J->slot[s];
 248     if (tref_isinteger(tr)) {
 249       IRIns *ir = IR(tref_ref(tr));
 250       if (!(ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_READONLY)))
 251         J->slot[s] = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);
 252     }
 253   }
 254 }
 255 
 256 /* Stop recording. */
 257 void lj_record_stop(jit_State *J, TraceLink linktype, TraceNo lnk)
 258 {
 259 #ifdef LUAJIT_ENABLE_TABLE_BUMP
 260   if (J->retryrec)
 261     lj_trace_err(J, LJ_TRERR_RETRY);
 262 #endif
 263   lj_trace_end(J);
 264   J->cur.linktype = (uint8_t)linktype;
 265   J->cur.link = (uint16_t)lnk;
 266   /* Looping back at the same stack level? */
 267   if (lnk == J->cur.traceno && J->framedepth + J->retdepth == 0) {
 268     if ((J->flags & JIT_F_OPT_LOOP))  /* Shall we try to create a loop? */
 269       goto nocanon;  /* Do not canonicalize or we lose the narrowing. */
 270     if (J->cur.root)  /* Otherwise ensure we always link to the root trace. */
 271       J->cur.link = J->cur.root;
 272   }
 273   canonicalize_slots(J);
 274 nocanon:
 275   /* Note: all loop ops must set J->pc to the following instruction! */
 276   lj_snap_add(J);  /* Add loop snapshot. */
 277   J->needsnap = 0;
 278   J->mergesnap = 1;  /* In case recording continues. */
 279 }
 280 
 281 /* Search bytecode backwards for a int/num constant slot initializer. */
 282 static TRef find_kinit(jit_State *J, const BCIns *endpc, BCReg slot, IRType t)
 283 {
 284   /* This algorithm is rather simplistic and assumes quite a bit about
 285   ** how the bytecode is generated. It works fine for FORI initializers,
 286   ** but it won't necessarily work in other cases (e.g. iterator arguments).
 287   ** It doesn't do anything fancy, either (like backpropagating MOVs).
 288   */
 289   const BCIns *pc, *startpc = proto_bc(J->pt);
 290   for (pc = endpc-1; pc > startpc; pc--) {
 291     BCIns ins = *pc;
 292     BCOp op = bc_op(ins);
 293     /* First try to find the last instruction that stores to this slot. */
 294     if (bcmode_a(op) == BCMbase && bc_a(ins) <= slot) {
 295       return 0;  /* Multiple results, e.g. from a CALL or KNIL. */
 296     } else if (bcmode_a(op) == BCMdst && bc_a(ins) == slot) {
 297       if (op == BC_KSHORT || op == BC_KNUM) {  /* Found const. initializer. */
 298         /* Now try to verify there's no forward jump across it. */
 299         const BCIns *kpc = pc;
 300         for (; pc > startpc; pc--)
 301           if (bc_op(*pc) == BC_JMP) {
 302             const BCIns *target = pc+bc_j(*pc)+1;
 303             if (target > kpc && target <= endpc)
 304               return 0;  /* Conditional assignment. */
 305           }
 306         if (op == BC_KSHORT) {
 307           int32_t k = (int32_t)(int16_t)bc_d(ins);
 308           return t == IRT_INT ? lj_ir_kint(J, k) : lj_ir_knum(J, (lua_Number)k);
 309         } else {
 310           cTValue *tv = proto_knumtv(J->pt, bc_d(ins));
 311           if (t == IRT_INT) {
 312             int32_t k = numberVint(tv);
 313             if (tvisint(tv) || numV(tv) == (lua_Number)k)  /* -0 is ok here. */
 314               return lj_ir_kint(J, k);
 315             return 0;  /* Type mismatch. */
 316           } else {
 317             return lj_ir_knum(J, numberVnum(tv));
 318           }
 319         }
 320       }
 321       return 0;  /* Non-constant initializer. */
 322     }
 323   }
 324   return 0;  /* No assignment to this slot found? */
 325 }
 326 
 327 /* Load and optionally convert a FORI argument from a slot. */
 328 static TRef fori_load(jit_State *J, BCReg slot, IRType t, int mode)
 329 {
 330   int conv = (tvisint(&J->L->base[slot]) != (t==IRT_INT)) ? IRSLOAD_CONVERT : 0;
 331   return sloadt(J, (int32_t)slot,
 332                 t + (((mode & IRSLOAD_TYPECHECK) ||
 333                       (conv && t == IRT_INT && !(mode >> 16))) ?
 334                      IRT_GUARD : 0),
 335                 mode + conv);
 336 }
 337 
 338 /* Peek before FORI to find a const initializer. Otherwise load from slot. */
 339 static TRef fori_arg(jit_State *J, const BCIns *fori, BCReg slot,
 340                      IRType t, int mode)
 341 {
 342   TRef tr = J->base[slot];
 343   if (!tr) {
 344     tr = find_kinit(J, fori, slot, t);
 345     if (!tr)
 346       tr = fori_load(J, slot, t, mode);
 347   }
 348   return tr;
 349 }
 350 
 351 /* Return the direction of the FOR loop iterator.
 352 ** It's important to exactly reproduce the semantics of the interpreter.
 353 */
 354 static int rec_for_direction(cTValue *o)
 355 {
 356   return (tvisint(o) ? intV(o) : (int32_t)o->u32.hi) >= 0;
 357 }
 358 
 359 /* Simulate the runtime behavior of the FOR loop iterator. */
 360 static LoopEvent rec_for_iter(IROp *op, cTValue *o, int isforl)
 361 {
 362   lua_Number stopv = numberVnum(&o[FORL_STOP]);
 363   lua_Number idxv = numberVnum(&o[FORL_IDX]);
 364   lua_Number stepv = numberVnum(&o[FORL_STEP]);
 365   if (isforl)
 366     idxv += stepv;
 367   if (rec_for_direction(&o[FORL_STEP])) {
 368     if (idxv <= stopv) {
 369       *op = IR_LE;
 370       return idxv + 2*stepv > stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER;
 371     }
 372     *op = IR_GT; return LOOPEV_LEAVE;
 373   } else {
 374     if (stopv <= idxv) {
 375       *op = IR_GE;
 376       return idxv + 2*stepv < stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER;
 377     }
 378     *op = IR_LT; return LOOPEV_LEAVE;
 379   }
 380 }
 381 
 382 /* Record checks for FOR loop overflow and step direction. */
 383 static void rec_for_check(jit_State *J, IRType t, int dir,
 384                           TRef stop, TRef step, int init)
 385 {
 386   if (!tref_isk(step)) {
 387     /* Non-constant step: need a guard for the direction. */
 388     TRef zero = (t == IRT_INT) ? lj_ir_kint(J, 0) : lj_ir_knum_zero(J);
 389     emitir(IRTG(dir ? IR_GE : IR_LT, t), step, zero);
 390     /* Add hoistable overflow checks for a narrowed FORL index. */
 391     if (init && t == IRT_INT) {
 392       if (tref_isk(stop)) {
 393         /* Constant stop: optimize check away or to a range check for step. */
 394         int32_t k = IR(tref_ref(stop))->i;
 395         if (dir) {
 396           if (k > 0)
 397             emitir(IRTGI(IR_LE), step, lj_ir_kint(J, (int32_t)0x7fffffff-k));
 398         } else {
 399           if (k < 0)
 400             emitir(IRTGI(IR_GE), step, lj_ir_kint(J, (int32_t)0x80000000-k));
 401         }
 402       } else {
 403         /* Stop+step variable: need full overflow check. */
 404         TRef tr = emitir(IRTGI(IR_ADDOV), step, stop);
 405         emitir(IRTI(IR_USE), tr, 0);  /* ADDOV is weak. Avoid dead result. */
 406       }
 407     }
 408   } else if (init && t == IRT_INT && !tref_isk(stop)) {
 409     /* Constant step: optimize overflow check to a range check for stop. */
 410     int32_t k = IR(tref_ref(step))->i;
 411     k = (int32_t)(dir ? 0x7fffffff : 0x80000000) - k;
 412     emitir(IRTGI(dir ? IR_LE : IR_GE), stop, lj_ir_kint(J, k));
 413   }
 414 }
 415 
 416 /* Record a FORL instruction. */
 417 static void rec_for_loop(jit_State *J, const BCIns *fori, ScEvEntry *scev,
 418                          int init)
 419 {
 420   BCReg ra = bc_a(*fori);
 421   cTValue *tv = &J->L->base[ra];
 422   TRef idx = J->base[ra+FORL_IDX];
 423   IRType t = idx ? tref_type(idx) :
 424              (init || LJ_DUALNUM) ? lj_opt_narrow_forl(J, tv) : IRT_NUM;
 425   int mode = IRSLOAD_INHERIT +
 426     ((!LJ_DUALNUM || tvisint(tv) == (t == IRT_INT)) ? IRSLOAD_READONLY : 0);
 427   TRef stop = fori_arg(J, fori, ra+FORL_STOP, t, mode);
 428   TRef step = fori_arg(J, fori, ra+FORL_STEP, t, mode);
 429   int tc, dir = rec_for_direction(&tv[FORL_STEP]);
 430   lua_assert(bc_op(*fori) == BC_FORI || bc_op(*fori) == BC_JFORI);
 431   scev->t.irt = t;
 432   scev->dir = dir;
 433   scev->stop = tref_ref(stop);
 434   scev->step = tref_ref(step);
 435   rec_for_check(J, t, dir, stop, step, init);
 436   scev->start = tref_ref(find_kinit(J, fori, ra+FORL_IDX, IRT_INT));
 437   tc = (LJ_DUALNUM &&
 438         !(scev->start && irref_isk(scev->stop) && irref_isk(scev->step) &&
 439           tvisint(&tv[FORL_IDX]) == (t == IRT_INT))) ?
 440         IRSLOAD_TYPECHECK : 0;
 441   if (tc) {
 442     J->base[ra+FORL_STOP] = stop;
 443     J->base[ra+FORL_STEP] = step;
 444   }
 445   if (!idx)
 446     idx = fori_load(J, ra+FORL_IDX, t,
 447                     IRSLOAD_INHERIT + tc + (J->scev.start << 16));
 448   if (!init)
 449     J->base[ra+FORL_IDX] = idx = emitir(IRT(IR_ADD, t), idx, step);
 450   J->base[ra+FORL_EXT] = idx;
 451   scev->idx = tref_ref(idx);
 452   setmref(scev->pc, fori);
 453   J->maxslot = ra+FORL_EXT+1;
 454 }
 455 
 456 /* Record FORL/JFORL or FORI/JFORI. */
 457 static LoopEvent rec_for(jit_State *J, const BCIns *fori, int isforl)
 458 {
 459   BCReg ra = bc_a(*fori);
 460   TValue *tv = &J->L->base[ra];
 461   TRef *tr = &J->base[ra];
 462   IROp op;
 463   LoopEvent ev;
 464   TRef stop;
 465   IRType t;
 466   if (isforl) {  /* Handle FORL/JFORL opcodes. */
 467     TRef idx = tr[FORL_IDX];
 468     if (mref(J->scev.pc, const BCIns) == fori && tref_ref(idx) == J->scev.idx) {
 469       t = J->scev.t.irt;
 470       stop = J->scev.stop;
 471       idx = emitir(IRT(IR_ADD, t), idx, J->scev.step);
 472       tr[FORL_EXT] = tr[FORL_IDX] = idx;
 473     } else {
 474       ScEvEntry scev;
 475       rec_for_loop(J, fori, &scev, 0);
 476       t = scev.t.irt;
 477       stop = scev.stop;
 478     }
 479   } else {  /* Handle FORI/JFORI opcodes. */
 480     BCReg i;
 481     lj_meta_for(J->L, tv);
 482     t = (LJ_DUALNUM || tref_isint(tr[FORL_IDX])) ? lj_opt_narrow_forl(J, tv) :
 483                                                    IRT_NUM;
 484     for (i = FORL_IDX; i <= FORL_STEP; i++) {
 485       if (!tr[i]) sload(J, ra+i);
 486       lua_assert(tref_isnumber_str(tr[i]));
 487       if (tref_isstr(tr[i]))
 488         tr[i] = emitir(IRTG(IR_STRTO, IRT_NUM), tr[i], 0);
 489       if (t == IRT_INT) {
 490         if (!tref_isinteger(tr[i]))
 491           tr[i] = emitir(IRTGI(IR_CONV), tr[i], IRCONV_INT_NUM|IRCONV_CHECK);
 492       } else {
 493         if (!tref_isnum(tr[i]))
 494           tr[i] = emitir(IRTN(IR_CONV), tr[i], IRCONV_NUM_INT);
 495       }
 496     }
 497     tr[FORL_EXT] = tr[FORL_IDX];
 498     stop = tr[FORL_STOP];
 499     rec_for_check(J, t, rec_for_direction(&tv[FORL_STEP]),
 500                   stop, tr[FORL_STEP], 1);
 501   }
 502 
 503   ev = rec_for_iter(&op, tv, isforl);
 504   if (ev == LOOPEV_LEAVE) {
 505     J->maxslot = ra+FORL_EXT+1;
 506     J->pc = fori+1;
 507   } else {
 508     J->maxslot = ra;
 509     J->pc = fori+bc_j(*fori)+1;
 510   }
 511   lj_snap_add(J);
 512 
 513   emitir(IRTG(op, t), tr[FORL_IDX], stop);
 514 
 515   if (ev == LOOPEV_LEAVE) {
 516     J->maxslot = ra;
 517     J->pc = fori+bc_j(*fori)+1;
 518   } else {
 519     J->maxslot = ra+FORL_EXT+1;
 520     J->pc = fori+1;
 521   }
 522   J->needsnap = 1;
 523   return ev;
 524 }
 525 
 526 /* Record ITERL/JITERL. */
 527 static LoopEvent rec_iterl(jit_State *J, const BCIns iterins)
 528 {
 529   BCReg ra = bc_a(iterins);
 530   if (!tref_isnil(getslot(J, ra))) {  /* Looping back? */
 531     J->base[ra-1] = J->base[ra];  /* Copy result of ITERC to control var. */
 532     J->maxslot = ra-1+bc_b(J->pc[-1]);
 533     J->pc += bc_j(iterins)+1;
 534     return LOOPEV_ENTER;
 535   } else {
 536     J->maxslot = ra-3;
 537     J->pc++;
 538     return LOOPEV_LEAVE;
 539   }
 540 }
 541 
 542 /* Record LOOP/JLOOP. Now, that was easy. */
 543 static LoopEvent rec_loop(jit_State *J, BCReg ra)
 544 {
 545   if (ra < J->maxslot) J->maxslot = ra;
 546   J->pc++;
 547   return LOOPEV_ENTER;
 548 }
 549 
 550 /* Check if a loop repeatedly failed to trace because it didn't loop back. */
 551 static int innerloopleft(jit_State *J, const BCIns *pc)
 552 {
 553   ptrdiff_t i;
 554   for (i = 0; i < PENALTY_SLOTS; i++)
 555     if (mref(J->penalty[i].pc, const BCIns) == pc) {
 556       if ((J->penalty[i].reason == LJ_TRERR_LLEAVE ||
 557            J->penalty[i].reason == LJ_TRERR_LINNER) &&
 558           J->penalty[i].val >= 2*PENALTY_MIN)
 559         return 1;
 560       break;
 561     }
 562   return 0;
 563 }
 564 
 565 /* Handle the case when an interpreted loop op is hit. */
 566 static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev)
 567 {
 568   if (J->parent == 0 && J->exitno == 0) {
 569     if (pc == J->startpc && J->framedepth + J->retdepth == 0) {
 570       /* Same loop? */
 571       if (ev == LOOPEV_LEAVE)  /* Must loop back to form a root trace. */
 572         lj_trace_err(J, LJ_TRERR_LLEAVE);
 573       lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno);  /* Looping trace. */
 574     } else if (ev != LOOPEV_LEAVE) {  /* Entering inner loop? */
 575       /* It's usually better to abort here and wait until the inner loop
 576       ** is traced. But if the inner loop repeatedly didn't loop back,
 577       ** this indicates a low trip count. In this case try unrolling
 578       ** an inner loop even in a root trace. But it's better to be a bit
 579       ** more conservative here and only do it for very short loops.
 580       */
 581       if (bc_j(*pc) != -1 && !innerloopleft(J, pc))
 582         lj_trace_err(J, LJ_TRERR_LINNER);  /* Root trace hit an inner loop. */
 583       if ((ev != LOOPEV_ENTERLO &&
 584            J->loopref && J->cur.nins - J->loopref > 24) || --J->loopunroll < 0)
 585         lj_trace_err(J, LJ_TRERR_LUNROLL);  /* Limit loop unrolling. */
 586       J->loopref = J->cur.nins;
 587     }
 588   } else if (ev != LOOPEV_LEAVE) {  /* Side trace enters an inner loop. */
 589     J->loopref = J->cur.nins;
 590     if (--J->loopunroll < 0)
 591       lj_trace_err(J, LJ_TRERR_LUNROLL);  /* Limit loop unrolling. */
 592   }  /* Side trace continues across a loop that's left or not entered. */
 593 }
 594 
 595 /* Handle the case when an already compiled loop op is hit. */
 596 static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev)
 597 {
 598   if (J->parent == 0 && J->exitno == 0) {  /* Root trace hit an inner loop. */
 599     /* Better let the inner loop spawn a side trace back here. */
 600     lj_trace_err(J, LJ_TRERR_LINNER);
 601   } else if (ev != LOOPEV_LEAVE) {  /* Side trace enters a compiled loop. */
 602     J->instunroll = 0;  /* Cannot continue across a compiled loop op. */
 603     if (J->pc == J->startpc && J->framedepth + J->retdepth == 0)
 604       lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno);  /* Form extra loop. */
 605     else
 606       lj_record_stop(J, LJ_TRLINK_ROOT, lnk);  /* Link to the loop. */
 607   }  /* Side trace continues across a loop that's left or not entered. */
 608 }
 609 
 610 /* -- Record profiler hook checks ----------------------------------------- */
 611 
 612 #if LJ_HASPROFILE
 613 
 614 /* Need to insert profiler hook check? */
 615 static int rec_profile_need(jit_State *J, GCproto *pt, const BCIns *pc)
 616 {
 617   GCproto *ppt;
 618   lua_assert(J->prof_mode == 'f' || J->prof_mode == 'l');
 619   if (!pt)
 620     return 0;
 621   ppt = J->prev_pt;
 622   J->prev_pt = pt;
 623   if (pt != ppt && ppt) {
 624     J->prev_line = -1;
 625     return 1;
 626   }
 627   if (J->prof_mode == 'l') {
 628     BCLine line = lj_debug_line(pt, proto_bcpos(pt, pc));
 629     BCLine pline = J->prev_line;
 630     J->prev_line = line;
 631     if (pline != line)
 632       return 1;
 633   }
 634   return 0;
 635 }
 636 
 637 static void rec_profile_ins(jit_State *J, const BCIns *pc)
 638 {
 639   if (J->prof_mode && rec_profile_need(J, J->pt, pc)) {
 640     emitir(IRTG(IR_PROF, IRT_NIL), 0, 0);
 641     lj_snap_add(J);
 642   }
 643 }
 644 
 645 static void rec_profile_ret(jit_State *J)
 646 {
 647   if (J->prof_mode == 'f') {
 648     emitir(IRTG(IR_PROF, IRT_NIL), 0, 0);
 649     J->prev_pt = NULL;
 650     lj_snap_add(J);
 651   }
 652 }
 653 
 654 #endif
 655 
 656 /* -- Record calls and returns -------------------------------------------- */
 657 
 658 /* Specialize to the runtime value of the called function or its prototype. */
 659 static TRef rec_call_specialize(jit_State *J, GCfunc *fn, TRef tr)
 660 {
 661   TRef kfunc;
 662   if (isluafunc(fn)) {
 663     GCproto *pt = funcproto(fn);
 664     /* Too many closures created? Probably not a monomorphic function. */
 665     if (pt->flags >= PROTO_CLC_POLY) {  /* Specialize to prototype instead. */
 666       TRef trpt = emitir(IRT(IR_FLOAD, IRT_PGC), tr, IRFL_FUNC_PC);
 667       emitir(IRTG(IR_EQ, IRT_PGC), trpt, lj_ir_kptr(J, proto_bc(pt)));
 668       (void)lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);  /* Prevent GC of proto. */
 669       return tr;
 670     }
 671   } else {
 672     /* Don't specialize to non-monomorphic builtins. */
 673     switch (fn->c.ffid) {
 674     case FF_coroutine_wrap_aux:
 675     case FF_string_gmatch_aux:
 676       /* NYI: io_file_iter doesn't have an ffid, yet. */
 677       {  /* Specialize to the ffid. */
 678         TRef trid = emitir(IRT(IR_FLOAD, IRT_U8), tr, IRFL_FUNC_FFID);
 679         emitir(IRTG(IR_EQ, IRT_INT), trid, lj_ir_kint(J, fn->c.ffid));
 680       }
 681       return tr;
 682     default:
 683       /* NYI: don't specialize to non-monomorphic C functions. */
 684       break;
 685     }
 686   }
 687   /* Otherwise specialize to the function (closure) value itself. */
 688   kfunc = lj_ir_kfunc(J, fn);
 689   emitir(IRTG(IR_EQ, IRT_FUNC), tr, kfunc);
 690   return kfunc;
 691 }
 692 
 693 /* Record call setup. */
 694 static void rec_call_setup(jit_State *J, BCReg func, ptrdiff_t nargs)
 695 {
 696   RecordIndex ix;
 697   TValue *functv = &J->L->base[func];
 698   TRef kfunc, *fbase = &J->base[func];
 699   ptrdiff_t i;
 700   (void)getslot(J, func); /* Ensure func has a reference. */
 701   for (i = 1; i <= nargs; i++)
 702     (void)getslot(J, func+LJ_FR2+i);  /* Ensure all args have a reference. */
 703   if (!tref_isfunc(fbase[0])) {  /* Resolve __call metamethod. */
 704     ix.tab = fbase[0];
 705     copyTV(J->L, &ix.tabv, functv);
 706     if (!lj_record_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj))
 707       lj_trace_err(J, LJ_TRERR_NOMM);
 708     for (i = ++nargs; i > LJ_FR2; i--)  /* Shift arguments up. */
 709       fbase[i+LJ_FR2] = fbase[i+LJ_FR2-1];
 710 #if LJ_FR2
 711     fbase[2] = fbase[0];
 712 #endif
 713     fbase[0] = ix.mobj;  /* Replace function. */
 714     functv = &ix.mobjv;
 715   }
 716   kfunc = rec_call_specialize(J, funcV(functv), fbase[0]);
 717 #if LJ_FR2
 718   fbase[0] = kfunc;
 719   fbase[1] = TREF_FRAME;
 720 #else
 721   fbase[0] = kfunc | TREF_FRAME;
 722 #endif
 723   J->maxslot = (BCReg)nargs;
 724 }
 725 
 726 /* Record call. */
 727 void lj_record_call(jit_State *J, BCReg func, ptrdiff_t nargs)
 728 {
 729   rec_call_setup(J, func, nargs);
 730   /* Bump frame. */
 731   J->framedepth++;
 732   J->base += func+1+LJ_FR2;
 733   J->baseslot += func+1+LJ_FR2;
 734   if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS)
 735     lj_trace_err(J, LJ_TRERR_STACKOV);
 736 }
 737 
 738 /* Record tail call. */
 739 void lj_record_tailcall(jit_State *J, BCReg func, ptrdiff_t nargs)
 740 {
 741   rec_call_setup(J, func, nargs);
 742   if (frame_isvarg(J->L->base - 1)) {
 743     BCReg cbase = (BCReg)frame_delta(J->L->base - 1);
 744     if (--J->framedepth < 0)
 745       lj_trace_err(J, LJ_TRERR_NYIRETL);
 746     J->baseslot -= (BCReg)cbase;
 747     J->base -= cbase;
 748     func += cbase;
 749   }
 750   /* Move func + args down. */
 751   if (LJ_FR2 && J->baseslot == 2)
 752     J->base[func+1] = TREF_FRAME;
 753   memmove(&J->base[-1-LJ_FR2], &J->base[func], sizeof(TRef)*(J->maxslot+1+LJ_FR2));
 754   /* Note: the new TREF_FRAME is now at J->base[-1] (even for slot #0). */
 755   /* Tailcalls can form a loop, so count towards the loop unroll limit. */
 756   if (++J->tailcalled > J->loopunroll)
 757     lj_trace_err(J, LJ_TRERR_LUNROLL);
 758 }
 759 
 760 /* Check unroll limits for down-recursion. */
 761 static int check_downrec_unroll(jit_State *J, GCproto *pt)
 762 {
 763   IRRef ptref;
 764   for (ptref = J->chain[IR_KGC]; ptref; ptref = IR(ptref)->prev)
 765     if (ir_kgc(IR(ptref)) == obj2gco(pt)) {
 766       int count = 0;
 767       IRRef ref;
 768       for (ref = J->chain[IR_RETF]; ref; ref = IR(ref)->prev)
 769         if (IR(ref)->op1 == ptref)
 770           count++;
 771       if (count) {
 772         if (J->pc == J->startpc) {
 773           if (count + J->tailcalled > J->param[JIT_P_recunroll])
 774             return 1;
 775         } else {
 776           lj_trace_err(J, LJ_TRERR_DOWNREC);
 777         }
 778       }
 779     }
 780   return 0;
 781 }
 782 
 783 static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot);
 784 
 785 /* Record return. */
 786 void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
 787 {
 788   TValue *frame = J->L->base - 1;
 789   ptrdiff_t i;
 790   for (i = 0; i < gotresults; i++)
 791     (void)getslot(J, rbase+i);  /* Ensure all results have a reference. */
 792   while (frame_ispcall(frame)) {  /* Immediately resolve pcall() returns. */
 793     BCReg cbase = (BCReg)frame_delta(frame);
 794     if (--J->framedepth <= 0)
 795       lj_trace_err(J, LJ_TRERR_NYIRETL);
 796     lua_assert(J->baseslot > 1+LJ_FR2);
 797     gotresults++;
 798     rbase += cbase;
 799     J->baseslot -= (BCReg)cbase;
 800     J->base -= cbase;
 801     J->base[--rbase] = TREF_TRUE;  /* Prepend true to results. */
 802     frame = frame_prevd(frame);
 803   }
 804   /* Return to lower frame via interpreter for unhandled cases. */
 805   if (J->framedepth == 0 && J->pt && bc_isret(bc_op(*J->pc)) &&
 806        (!frame_islua(frame) ||
 807         (J->parent == 0 && J->exitno == 0 &&
 808          !bc_isret(bc_op(J->cur.startins))))) {
 809     /* NYI: specialize to frame type and return directly, not via RET*. */
 810     for (i = 0; i < (ptrdiff_t)rbase; i++)
 811       J->base[i] = 0;  /* Purge dead slots. */
 812     J->maxslot = rbase + (BCReg)gotresults;
 813     lj_record_stop(J, LJ_TRLINK_RETURN, 0);  /* Return to interpreter. */
 814     return;
 815   }
 816   if (frame_isvarg(frame)) {
 817     BCReg cbase = (BCReg)frame_delta(frame);
 818     if (--J->framedepth < 0)  /* NYI: return of vararg func to lower frame. */
 819       lj_trace_err(J, LJ_TRERR_NYIRETL);
 820     lua_assert(J->baseslot > 1+LJ_FR2);
 821     rbase += cbase;
 822     J->baseslot -= (BCReg)cbase;
 823     J->base -= cbase;
 824     frame = frame_prevd(frame);
 825   }
 826   if (frame_islua(frame)) {  /* Return to Lua frame. */
 827     BCIns callins = *(frame_pc(frame)-1);
 828     ptrdiff_t nresults = bc_b(callins) ? (ptrdiff_t)bc_b(callins)-1 :gotresults;
 829     BCReg cbase = bc_a(callins);
 830     GCproto *pt = funcproto(frame_func(frame - (cbase+1+LJ_FR2)));
 831     if ((pt->flags & PROTO_NOJIT))
 832       lj_trace_err(J, LJ_TRERR_CJITOFF);
 833     if (J->framedepth == 0 && J->pt && frame == J->L->base - 1) {
 834       if (check_downrec_unroll(J, pt)) {
 835         J->maxslot = (BCReg)(rbase + gotresults);
 836         lj_snap_purge(J);
 837         lj_record_stop(J, LJ_TRLINK_DOWNREC, J->cur.traceno);  /* Down-rec. */
 838         return;
 839       }
 840       lj_snap_add(J);
 841     }
 842     for (i = 0; i < nresults; i++)  /* Adjust results. */
 843       J->base[i-1-LJ_FR2] = i < gotresults ? J->base[rbase+i] : TREF_NIL;
 844     J->maxslot = cbase+(BCReg)nresults;
 845     if (J->framedepth > 0) {  /* Return to a frame that is part of the trace. */
 846       J->framedepth--;
 847       lua_assert(J->baseslot > cbase+1+LJ_FR2);
 848       J->baseslot -= cbase+1+LJ_FR2;
 849       J->base -= cbase+1+LJ_FR2;
 850     } else if (J->parent == 0 && J->exitno == 0 &&
 851                !bc_isret(bc_op(J->cur.startins))) {
 852       /* Return to lower frame would leave the loop in a root trace. */
 853       lj_trace_err(J, LJ_TRERR_LLEAVE);
 854     } else if (J->needsnap) {  /* Tailcalled to ff with side-effects. */
 855       lj_trace_err(J, LJ_TRERR_NYIRETL);  /* No way to insert snapshot here. */
 856     } else {  /* Return to lower frame. Guard for the target we return to. */
 857       TRef trpt = lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);
 858       TRef trpc = lj_ir_kptr(J, (void *)frame_pc(frame));
 859       emitir(IRTG(IR_RETF, IRT_PGC), trpt, trpc);
 860       J->retdepth++;
 861       J->needsnap = 1;
 862       lua_assert(J->baseslot == 1+LJ_FR2);
 863       /* Shift result slots up and clear the slots of the new frame below. */
 864       memmove(J->base + cbase, J->base-1-LJ_FR2, sizeof(TRef)*nresults);
 865       memset(J->base-1-LJ_FR2, 0, sizeof(TRef)*(cbase+1+LJ_FR2));
 866     }
 867   } else if (frame_iscont(frame)) {  /* Return to continuation frame. */
 868     ASMFunction cont = frame_contf(frame);
 869     BCReg cbase = (BCReg)frame_delta(frame);
 870     if ((J->framedepth -= 2) < 0)
 871       lj_trace_err(J, LJ_TRERR_NYIRETL);
 872     J->baseslot -= (BCReg)cbase;
 873     J->base -= cbase;
 874     J->maxslot = cbase-(2<<LJ_FR2);
 875     if (cont == lj_cont_ra) {
 876       /* Copy result to destination slot. */
 877       BCReg dst = bc_a(*(frame_contpc(frame)-1));
 878       J->base[dst] = gotresults ? J->base[cbase+rbase] : TREF_NIL;
 879       if (dst >= J->maxslot) {
 880         J->maxslot = dst+1;
 881       }
 882     } else if (cont == lj_cont_nop) {
 883       /* Nothing to do here. */
 884     } else if (cont == lj_cont_cat) {
 885       BCReg bslot = bc_b(*(frame_contpc(frame)-1));
 886       TRef tr = gotresults ? J->base[cbase+rbase] : TREF_NIL;
 887       if (bslot != J->maxslot) {  /* Concatenate the remainder. */
 888         TValue *b = J->L->base, save;  /* Simulate lower frame and result. */
 889         J->base[J->maxslot] = tr;
 890         copyTV(J->L, &save, b-(2<<LJ_FR2));
 891         if (gotresults)
 892           copyTV(J->L, b-(2<<LJ_FR2), b+rbase);
 893         else
 894           setnilV(b-(2<<LJ_FR2));
 895         J->L->base = b - cbase;
 896         tr = rec_cat(J, bslot, cbase-(2<<LJ_FR2));
 897         b = J->L->base + cbase;  /* Undo. */
 898         J->L->base = b;
 899         copyTV(J->L, b-(2<<LJ_FR2), &save);
 900       }
 901       if (tr) {  /* Store final result. */
 902         BCReg dst = bc_a(*(frame_contpc(frame)-1));
 903         J->base[dst] = tr;
 904         if (dst >= J->maxslot) {
 905           J->maxslot = dst+1;
 906         }
 907       }  /* Otherwise continue with another __concat call. */
 908     } else {
 909       /* Result type already specialized. */
 910       lua_assert(cont == lj_cont_condf || cont == lj_cont_condt);
 911     }
 912   } else {
 913     lj_trace_err(J, LJ_TRERR_NYIRETL);  /* NYI: handle return to C frame. */
 914   }
 915   lua_assert(J->baseslot >= 1+LJ_FR2);
 916 }
 917 
 918 /* -- Metamethod handling ------------------------------------------------- */
 919 
 920 /* Prepare to record call to metamethod. */
 921 static BCReg rec_mm_prep(jit_State *J, ASMFunction cont)
 922 {
 923   BCReg s, top = cont == lj_cont_cat ? J->maxslot : curr_proto(J->L)->framesize;
 924 #if LJ_FR2
 925   J->base[top] = lj_ir_k64(J, IR_KNUM, u64ptr(contptr(cont)));
 926   J->base[top+1] = TREF_CONT;
 927 #else
 928   J->base[top] = lj_ir_kptr(J, contptr(cont)) | TREF_CONT;
 929 #endif
 930   J->framedepth++;
 931   for (s = J->maxslot; s < top; s++)
 932     J->base[s] = 0;  /* Clear frame gap to avoid resurrecting previous refs. */
 933   return top+1+LJ_FR2;
 934 }
 935 
 936 /* Record metamethod lookup. */
 937 int lj_record_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm)
 938 {
 939   RecordIndex mix;
 940   GCtab *mt;
 941   if (tref_istab(ix->tab)) {
 942     mt = tabref(tabV(&ix->tabv)->metatable);
 943     mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META);
 944   } else if (tref_isudata(ix->tab)) {
 945     int udtype = udataV(&ix->tabv)->udtype;
 946     mt = tabref(udataV(&ix->tabv)->metatable);
 947     /* The metatables of special userdata objects are treated as immutable. */
 948     if (udtype != UDTYPE_USERDATA) {
 949       cTValue *mo;
 950       if (LJ_HASFFI && udtype == UDTYPE_FFI_CLIB) {
 951         /* Specialize to the C library namespace object. */
 952         emitir(IRTG(IR_EQ, IRT_PGC), ix->tab, lj_ir_kptr(J, udataV(&ix->tabv)));
 953       } else {
 954         /* Specialize to the type of userdata. */
 955         TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), ix->tab, IRFL_UDATA_UDTYPE);
 956         emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, udtype));
 957       }
 958   immutable_mt:
 959       mo = lj_tab_getstr(mt, mmname_str(J2G(J), mm));
 960       if (!mo || tvisnil(mo))
 961         return 0;  /* No metamethod. */
 962       /* Treat metamethod or index table as immutable, too. */
 963       if (!(tvisfunc(mo) || tvistab(mo)))
 964         lj_trace_err(J, LJ_TRERR_BADTYPE);
 965       copyTV(J->L, &ix->mobjv, mo);
 966       ix->mobj = lj_ir_kgc(J, gcV(mo), tvisfunc(mo) ? IRT_FUNC : IRT_TAB);
 967       ix->mtv = mt;
 968       ix->mt = TREF_NIL;  /* Dummy value for comparison semantics. */
 969       return 1;  /* Got metamethod or index table. */
 970     }
 971     mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_UDATA_META);
 972   } else {
 973     /* Specialize to base metatable. Must flush mcode in lua_setmetatable(). */
 974     mt = tabref(basemt_obj(J2G(J), &ix->tabv));
 975     if (mt == NULL) {
 976       ix->mt = TREF_NIL;
 977       return 0;  /* No metamethod. */
 978     }
 979     /* The cdata metatable is treated as immutable. */
 980     if (LJ_HASFFI && tref_iscdata(ix->tab)) goto immutable_mt;
 981 #if LJ_GC64
 982     /* TODO: fix ARM32 asm_fload(), so we can use this for all archs. */
 983     ix->mt = mix.tab = lj_ir_ggfload(J, IRT_TAB,
 984       GG_OFS(g.gcroot[GCROOT_BASEMT+itypemap(&ix->tabv)]));
 985 #else
 986     ix->mt = mix.tab = lj_ir_ktab(J, mt);
 987 #endif
 988     goto nocheck;
 989   }
 990   ix->mt = mt ? mix.tab : TREF_NIL;
 991   emitir(IRTG(mt ? IR_NE : IR_EQ, IRT_TAB), mix.tab, lj_ir_knull(J, IRT_TAB));
 992 nocheck:
 993   if (mt) {
 994     GCstr *mmstr = mmname_str(J2G(J), mm);
 995     cTValue *mo = lj_tab_getstr(mt, mmstr);
 996     if (mo && !tvisnil(mo))
 997       copyTV(J->L, &ix->mobjv, mo);
 998     ix->mtv = mt;
 999     settabV(J->L, &mix.tabv, mt);
1000     setstrV(J->L, &mix.keyv, mmstr);
1001     mix.key = lj_ir_kstr(J, mmstr);
1002     mix.val = 0;
1003     mix.idxchain = 0;
1004     ix->mobj = lj_record_idx(J, &mix);
1005     return !tref_isnil(ix->mobj);  /* 1 if metamethod found, 0 if not. */
1006   }
1007   return 0;  /* No metamethod. */
1008 }
1009 
1010 /* Record call to arithmetic metamethod. */
1011 static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm)
1012 {
1013   /* Set up metamethod call first to save ix->tab and ix->tabv. */
1014   BCReg func = rec_mm_prep(J, mm == MM_concat ? lj_cont_cat : lj_cont_ra);
1015   TRef *base = J->base + func;
1016   TValue *basev = J->L->base + func;
1017   base[1+LJ_FR2] = ix->tab; base[2+LJ_FR2] = ix->key;
1018   copyTV(J->L, basev+1+LJ_FR2, &ix->tabv);
1019   copyTV(J->L, basev+2+LJ_FR2, &ix->keyv);
1020   if (!lj_record_mm_lookup(J, ix, mm)) {  /* Lookup mm on 1st operand. */
1021     if (mm != MM_unm) {
1022       ix->tab = ix->key;
1023       copyTV(J->L, &ix->tabv, &ix->keyv);
1024       if (lj_record_mm_lookup(J, ix, mm))  /* Lookup mm on 2nd operand. */
1025         goto ok;
1026     }
1027     lj_trace_err(J, LJ_TRERR_NOMM);
1028   }
1029 ok:
1030   base[0] = ix->mobj;
1031 #if LJ_FR2
1032   base[1] = 0;
1033 #endif
1034   copyTV(J->L, basev+0, &ix->mobjv);
1035   lj_record_call(J, func, 2);
1036   return 0;  /* No result yet. */
1037 }
1038 
1039 /* Record call to __len metamethod. */
1040 static TRef rec_mm_len(jit_State *J, TRef tr, TValue *tv)
1041 {
1042   RecordIndex ix;
1043   ix.tab = tr;
1044   copyTV(J->L, &ix.tabv, tv);
1045   if (lj_record_mm_lookup(J, &ix, MM_len)) {
1046     BCReg func = rec_mm_prep(J, lj_cont_ra);
1047     TRef *base = J->base + func;
1048     TValue *basev = J->L->base + func;
1049     base[0] = ix.mobj; copyTV(J->L, basev+0, &ix.mobjv);
1050     base += LJ_FR2;
1051     basev += LJ_FR2;
1052     base[1] = tr; copyTV(J->L, basev+1, tv);
1053 #if LJ_52
1054     base[2] = tr; copyTV(J->L, basev+2, tv);
1055 #else
1056     base[2] = TREF_NIL; setnilV(basev+2);
1057 #endif
1058     lj_record_call(J, func, 2);
1059   } else {
1060     if (LJ_52 && tref_istab(tr))
1061       return lj_ir_call(J, IRCALL_lj_tab_len, tr);
1062     lj_trace_err(J, LJ_TRERR_NOMM);
1063   }
1064   return 0;  /* No result yet. */
1065 }
1066 
1067 /* Call a comparison metamethod. */
1068 static void rec_mm_callcomp(jit_State *J, RecordIndex *ix, int op)
1069 {
1070   BCReg func = rec_mm_prep(J, (op&1) ? lj_cont_condf : lj_cont_condt);
1071   TRef *base = J->base + func + LJ_FR2;
1072   TValue *tv = J->L->base + func + LJ_FR2;
1073   base[-LJ_FR2] = ix->mobj; base[1] = ix->val; base[2] = ix->key;
1074   copyTV(J->L, tv-LJ_FR2, &ix->mobjv);
1075   copyTV(J->L, tv+1, &ix->valv);
1076   copyTV(J->L, tv+2, &ix->keyv);
1077   lj_record_call(J, func, 2);
1078 }
1079 
1080 /* Record call to equality comparison metamethod (for tab and udata only). */
1081 static void rec_mm_equal(jit_State *J, RecordIndex *ix, int op)
1082 {
1083   ix->tab = ix->val;
1084   copyTV(J->L, &ix->tabv, &ix->valv);
1085   if (lj_record_mm_lookup(J, ix, MM_eq)) {  /* Lookup mm on 1st operand. */
1086     cTValue *bv;
1087     TRef mo1 = ix->mobj;
1088     TValue mo1v;
1089     copyTV(J->L, &mo1v, &ix->mobjv);
1090     /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
1091     bv = &ix->keyv;
1092     if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) {
1093       TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META);
1094       emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1095     } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) {
1096       TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META);
1097       emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1098     } else {  /* Lookup metamethod on 2nd operand and compare both. */
1099       ix->tab = ix->key;
1100       copyTV(J->L, &ix->tabv, bv);
1101       if (!lj_record_mm_lookup(J, ix, MM_eq) ||
1102           lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv))
1103         return;
1104     }
1105     rec_mm_callcomp(J, ix, op);
1106   }
1107 }
1108 
1109 /* Record call to ordered comparison metamethods (for arbitrary objects). */
1110 static void rec_mm_comp(jit_State *J, RecordIndex *ix, int op)
1111 {
1112   ix->tab = ix->val;
1113   copyTV(J->L, &ix->tabv, &ix->valv);
1114   while (1) {
1115     MMS mm = (op & 2) ? MM_le : MM_lt;  /* Try __le + __lt or only __lt. */
1116 #if LJ_52
1117     if (!lj_record_mm_lookup(J, ix, mm)) {  /* Lookup mm on 1st operand. */
1118       ix->tab = ix->key;
1119       copyTV(J->L, &ix->tabv, &ix->keyv);
1120       if (!lj_record_mm_lookup(J, ix, mm))  /* Lookup mm on 2nd operand. */
1121         goto nomatch;
1122     }
1123     rec_mm_callcomp(J, ix, op);
1124     return;
1125 #else
1126     if (lj_record_mm_lookup(J, ix, mm)) {  /* Lookup mm on 1st operand. */
1127       cTValue *bv;
1128       TRef mo1 = ix->mobj;
1129       TValue mo1v;
1130       copyTV(J->L, &mo1v, &ix->mobjv);
1131       /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
1132       bv = &ix->keyv;
1133       if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) {
1134         TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META);
1135         emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1136       } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) {
1137         TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META);
1138         emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1139       } else {  /* Lookup metamethod on 2nd operand and compare both. */
1140         ix->tab = ix->key;
1141         copyTV(J->L, &ix->tabv, bv);
1142         if (!lj_record_mm_lookup(J, ix, mm) ||
1143             lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv))
1144           goto nomatch;
1145       }
1146       rec_mm_callcomp(J, ix, op);
1147       return;
1148     }
1149 #endif
1150   nomatch:
1151     /* Lookup failed. Retry with  __lt and swapped operands. */
1152     if (!(op & 2)) break;  /* Already at __lt. Interpreter will throw. */
1153     ix->tab = ix->key; ix->key = ix->val; ix->val = ix->tab;
1154     copyTV(J->L, &ix->tabv, &ix->keyv);
1155     copyTV(J->L, &ix->keyv, &ix->valv);
1156     copyTV(J->L, &ix->valv, &ix->tabv);
1157     op ^= 3;
1158   }
1159 }
1160 
1161 #if LJ_HASFFI
1162 /* Setup call to cdata comparison metamethod. */
1163 static void rec_mm_comp_cdata(jit_State *J, RecordIndex *ix, int op, MMS mm)
1164 {
1165   lj_snap_add(J);
1166   if (tref_iscdata(ix->val)) {
1167     ix->tab = ix->val;
1168     copyTV(J->L, &ix->tabv, &ix->valv);
1169   } else {
1170     lua_assert(tref_iscdata(ix->key));
1171     ix->tab = ix->key;
1172     copyTV(J->L, &ix->tabv, &ix->keyv);
1173   }
1174   lj_record_mm_lookup(J, ix, mm);
1175   rec_mm_callcomp(J, ix, op);
1176 }
1177 #endif
1178 
1179 /* -- Indexed access ------------------------------------------------------ */
1180 
1181 #ifdef LUAJIT_ENABLE_TABLE_BUMP
1182 /* Bump table allocations in bytecode when they grow during recording. */
1183 static void rec_idx_bump(jit_State *J, RecordIndex *ix)
1184 {
1185   RBCHashEntry *rbc = &J->rbchash[(ix->tab & (RBCHASH_SLOTS-1))];
1186   if (tref_ref(ix->tab) == rbc->ref) {
1187     const BCIns *pc = mref(rbc->pc, const BCIns);
1188     GCtab *tb = tabV(&ix->tabv);
1189     uint32_t nhbits;
1190     IRIns *ir;
1191     if (!tvisnil(&ix->keyv))
1192       (void)lj_tab_set(J->L, tb, &ix->keyv);  /* Grow table right now. */
1193     nhbits = tb->hmask > 0 ? lj_fls(tb->hmask)+1 : 0;
1194     ir = IR(tref_ref(ix->tab));
1195     if (ir->o == IR_TNEW) {
1196       uint32_t ah = bc_d(*pc);
1197       uint32_t asize = ah & 0x7ff, hbits = ah >> 11;
1198       if (nhbits > hbits) hbits = nhbits;
1199       if (tb->asize > asize) {
1200         asize = tb->asize <= 0x7ff ? tb->asize : 0x7ff;
1201       }
1202       if ((asize | (hbits<<11)) != ah) {  /* Has the size changed? */
1203         /* Patch bytecode, but continue recording (for more patching). */
1204         setbc_d(pc, (asize | (hbits<<11)));
1205         /* Patching TNEW operands is only safe if the trace is aborted. */
1206         ir->op1 = asize; ir->op2 = hbits;
1207         J->retryrec = 1;  /* Abort the trace at the end of recording. */
1208       }
1209     } else if (ir->o == IR_TDUP) {
1210       GCtab *tpl = gco2tab(proto_kgc(&gcref(rbc->pt)->pt, ~(ptrdiff_t)bc_d(*pc)));
1211       /* Grow template table, but preserve keys with nil values. */
1212       if ((tb->asize > tpl->asize && (1u << nhbits)-1 == tpl->hmask) ||
1213           (tb->asize == tpl->asize && (1u << nhbits)-1 > tpl->hmask)) {
1214         Node *node = noderef(tpl->node);
1215         uint32_t i, hmask = tpl->hmask, asize;
1216         TValue *array;
1217         for (i = 0; i <= hmask; i++) {
1218           if (!tvisnil(&node[i].key) && tvisnil(&node[i].val))
1219             settabV(J->L, &node[i].val, tpl);
1220         }
1221         if (!tvisnil(&ix->keyv) && tref_isk(ix->key)) {
1222           TValue *o = lj_tab_set(J->L, tpl, &ix->keyv);
1223           if (tvisnil(o)) settabV(J->L, o, tpl);
1224         }
1225         lj_tab_resize(J->L, tpl, tb->asize, nhbits);
1226         node = noderef(tpl->node);
1227         hmask = tpl->hmask;
1228         for (i = 0; i <= hmask; i++) {
1229           /* This is safe, since template tables only hold immutable values. */
1230           if (tvistab(&node[i].val))
1231             setnilV(&node[i].val);
1232         }
1233         /* The shape of the table may have changed. Clean up array part, too. */
1234         asize = tpl->asize;
1235         array = tvref(tpl->array);
1236         for (i = 0; i < asize; i++) {
1237           if (tvistab(&array[i]))
1238             setnilV(&array[i]);
1239         }
1240         J->retryrec = 1;  /* Abort the trace at the end of recording. */
1241       }
1242     }
1243   }
1244 }
1245 #endif
1246 
1247 /* Record bounds-check. */
1248 static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize)
1249 {
1250   /* Try to emit invariant bounds checks. */
1251   if ((J->flags & (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) ==
1252       (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) {
1253     IRRef ref = tref_ref(ikey);
1254     IRIns *ir = IR(ref);
1255     int32_t ofs = 0;
1256     IRRef ofsref = 0;
1257     /* Handle constant offsets. */
1258     if (ir->o == IR_ADD && irref_isk(ir->op2)) {
1259       ofsref = ir->op2;
1260       ofs = IR(ofsref)->i;
1261       ref = ir->op1;
1262       ir = IR(ref);
1263     }
1264     /* Got scalar evolution analysis results for this reference? */
1265     if (ref == J->scev.idx) {
1266       int32_t stop;
1267       lua_assert(irt_isint(J->scev.t) && ir->o == IR_SLOAD);
1268       stop = numberVint(&(J->L->base - J->baseslot)[ir->op1 + FORL_STOP]);
1269       /* Runtime value for stop of loop is within bounds? */
1270       if ((uint64_t)stop + ofs < (uint64_t)asize) {
1271         /* Emit invariant bounds check for stop. */
1272         emitir(IRTG(IR_ABC, IRT_P32), asizeref, ofs == 0 ? J->scev.stop :
1273                emitir(IRTI(IR_ADD), J->scev.stop, ofsref));
1274         /* Emit invariant bounds check for start, if not const or negative. */
1275         if (!(J->scev.dir && J->scev.start &&
1276               (int64_t)IR(J->scev.start)->i + ofs >= 0))
1277           emitir(IRTG(IR_ABC, IRT_P32), asizeref, ikey);
1278         return;
1279       }
1280     }
1281   }
1282   emitir(IRTGI(IR_ABC), asizeref, ikey);  /* Emit regular bounds check. */
1283 }
1284 
1285 /* Record indexed key lookup. */
1286 static TRef rec_idx_key(jit_State *J, RecordIndex *ix, IRRef *rbref,
1287                         IRType1 *rbguard)
1288 {
1289   TRef key;
1290   GCtab *t = tabV(&ix->tabv);
1291   ix->oldv = lj_tab_get(J->L, t, &ix->keyv);  /* Lookup previous value. */
1292   *rbref = 0;
1293   rbguard->irt = 0;
1294 
1295   /* Integer keys are looked up in the array part first. */
1296   key = ix->key;
1297   if (tref_isnumber(key)) {
1298     int32_t k = numberVint(&ix->keyv);
1299     if (!tvisint(&ix->keyv) && numV(&ix->keyv) != (lua_Number)k)
1300       k = LJ_MAX_ASIZE;
1301     if ((MSize)k < LJ_MAX_ASIZE) {  /* Potential array key? */
1302       TRef ikey = lj_opt_narrow_index(J, key);
1303       TRef asizeref = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE);
1304       if ((MSize)k < t->asize) {  /* Currently an array key? */
1305         TRef arrayref;
1306         rec_idx_abc(J, asizeref, ikey, t->asize);
1307         arrayref = emitir(IRT(IR_FLOAD, IRT_PGC), ix->tab, IRFL_TAB_ARRAY);
1308         return emitir(IRT(IR_AREF, IRT_PGC), arrayref, ikey);
1309       } else {  /* Currently not in array (may be an array extension)? */
1310         emitir(IRTGI(IR_ULE), asizeref, ikey);  /* Inv. bounds check. */
1311         if (k == 0 && tref_isk(key))
1312           key = lj_ir_knum_zero(J);  /* Canonicalize 0 or +-0.0 to +0.0. */
1313         /* And continue with the hash lookup. */
1314       }
1315     } else if (!tref_isk(key)) {
1316       /* We can rule out const numbers which failed the integerness test
1317       ** above. But all other numbers are potential array keys.
1318       */
1319       if (t->asize == 0) {  /* True sparse tables have an empty array part. */
1320         /* Guard that the array part stays empty. */
1321         TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE);
1322         emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0));
1323       } else {
1324         lj_trace_err(J, LJ_TRERR_NYITMIX);
1325       }
1326     }
1327   }
1328 
1329   /* Otherwise the key is located in the hash part. */
1330   if (t->hmask == 0) {  /* Shortcut for empty hash part. */
1331     /* Guard that the hash part stays empty. */
1332     TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK);
1333     emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0));
1334     return lj_ir_kkptr(J, niltvg(J2G(J)));
1335   }
1336   if (tref_isinteger(key))  /* Hash keys are based on numbers, not ints. */
1337     key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
1338   if (tref_isk(key)) {
1339     /* Optimize lookup of constant hash keys. */
1340     MSize hslot = (MSize)((char *)ix->oldv - (char *)&noderef(t->node)[0].val);
1341     if (t->hmask > 0 && hslot <= t->hmask*(MSize)sizeof(Node) &&
1342         hslot <= 65535*(MSize)sizeof(Node)) {
1343       TRef node, kslot, hm;
1344       *rbref = J->cur.nins;  /* Mark possible rollback point. */
1345       *rbguard = J->guardemit;
1346       hm = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK);
1347       emitir(IRTGI(IR_EQ), hm, lj_ir_kint(J, (int32_t)t->hmask));
1348       node = emitir(IRT(IR_FLOAD, IRT_PGC), ix->tab, IRFL_TAB_NODE);
1349       kslot = lj_ir_kslot(J, key, hslot / sizeof(Node));
1350       return emitir(IRTG(IR_HREFK, IRT_PGC), node, kslot);
1351     }
1352   }
1353   /* Fall back to a regular hash lookup. */
1354   return emitir(IRT(IR_HREF, IRT_PGC), ix->tab, key);
1355 }
1356 
1357 /* Determine whether a key is NOT one of the fast metamethod names. */
1358 static int nommstr(jit_State *J, TRef key)
1359 {
1360   if (tref_isstr(key)) {
1361     if (tref_isk(key)) {
1362       GCstr *str = ir_kstr(IR(tref_ref(key)));
1363       uint32_t mm;
1364       for (mm = 0; mm <= MM_FAST; mm++)
1365         if (mmname_str(J2G(J), mm) == str)
1366           return 0;  /* MUST be one the fast metamethod names. */
1367     } else {
1368       return 0;  /* Variable string key MAY be a metamethod name. */
1369     }
1370   }
1371   return 1;  /* CANNOT be a metamethod name. */
1372 }
1373 
1374 /* Record indexed load/store. */
1375 TRef lj_record_idx(jit_State *J, RecordIndex *ix)
1376 {
1377   TRef xref;
1378   IROp xrefop, loadop;
1379   IRRef rbref;
1380   IRType1 rbguard;
1381   cTValue *oldv;
1382 
1383   while (!tref_istab(ix->tab)) { /* Handle non-table lookup. */
1384     /* Never call raw lj_record_idx() on non-table. */
1385     lua_assert(ix->idxchain != 0);
1386     if (!lj_record_mm_lookup(J, ix, ix->val ? MM_newindex : MM_index))
1387       lj_trace_err(J, LJ_TRERR_NOMM);
1388   handlemm:
1389     if (tref_isfunc(ix->mobj)) {  /* Handle metamethod call. */
1390       BCReg func = rec_mm_prep(J, ix->val ? lj_cont_nop : lj_cont_ra);
1391       TRef *base = J->base + func + LJ_FR2;
1392       TValue *tv = J->L->base + func + LJ_FR2;
1393       base[-LJ_FR2] = ix->mobj; base[1] = ix->tab; base[2] = ix->key;
1394       setfuncV(J->L, tv-LJ_FR2, funcV(&ix->mobjv));
1395       copyTV(J->L, tv+1, &ix->tabv);
1396       copyTV(J->L, tv+2, &ix->keyv);
1397       if (ix->val) {
1398         base[3] = ix->val;
1399         copyTV(J->L, tv+3, &ix->valv);
1400         lj_record_call(J, func, 3);  /* mobj(tab, key, val) */
1401         return 0;
1402       } else {
1403         lj_record_call(J, func, 2);  /* res = mobj(tab, key) */
1404         return 0;  /* No result yet. */
1405       }
1406     }
1407     /* Otherwise retry lookup with metaobject. */
1408     ix->tab = ix->mobj;
1409     copyTV(J->L, &ix->tabv, &ix->mobjv);
1410     if (--ix->idxchain == 0)
1411       lj_trace_err(J, LJ_TRERR_IDXLOOP);
1412   }
1413 
1414   /* First catch nil and NaN keys for tables. */
1415   if (tvisnil(&ix->keyv) || (tvisnum(&ix->keyv) && tvisnan(&ix->keyv))) {
1416     if (ix->val)  /* Better fail early. */
1417       lj_trace_err(J, LJ_TRERR_STORENN);
1418     if (tref_isk(ix->key)) {
1419       if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_index))
1420         goto handlemm;
1421       return TREF_NIL;
1422     }
1423   }
1424 
1425   /* Record the key lookup. */
1426   xref = rec_idx_key(J, ix, &rbref, &rbguard);
1427   xrefop = IR(tref_ref(xref))->o;
1428   loadop = xrefop == IR_AREF ? IR_ALOAD : IR_HLOAD;
1429   /* The lj_meta_tset() inconsistency is gone, but better play safe. */
1430   oldv = xrefop == IR_KKPTR ? (cTValue *)ir_kptr(IR(tref_ref(xref))) : ix->oldv;
1431 
1432   if (ix->val == 0) {  /* Indexed load */
1433     IRType t = itype2irt(oldv);
1434     TRef res;
1435     if (oldv == niltvg(J2G(J))) {
1436       emitir(IRTG(IR_EQ, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1437       res = TREF_NIL;
1438     } else {
1439       res = emitir(IRTG(loadop, t), xref, 0);
1440     }
1441     if (tref_ref(res) < rbref) {  /* HREFK + load forwarded? */
1442       lj_ir_rollback(J, rbref);  /* Rollback to eliminate hmask guard. */
1443       J->guardemit = rbguard;
1444     }
1445     if (t == IRT_NIL && ix->idxchain && lj_record_mm_lookup(J, ix, MM_index))
1446       goto handlemm;
1447     if (irtype_ispri(t)) res = TREF_PRI(t);  /* Canonicalize primitives. */
1448     return res;
1449   } else {  /* Indexed store. */
1450     GCtab *mt = tabref(tabV(&ix->tabv)->metatable);
1451     int keybarrier = tref_isgcv(ix->key) && !tref_isnil(ix->val);
1452     if (tref_ref(xref) < rbref) {  /* HREFK forwarded? */
1453       lj_ir_rollback(J, rbref);  /* Rollback to eliminate hmask guard. */
1454       J->guardemit = rbguard;
1455     }
1456     if (tvisnil(oldv)) {  /* Previous value was nil? */
1457       /* Need to duplicate the hasmm check for the early guards. */
1458       int hasmm = 0;
1459       if (ix->idxchain && mt) {
1460         cTValue *mo = lj_tab_getstr(mt, mmname_str(J2G(J), MM_newindex));
1461         hasmm = mo && !tvisnil(mo);
1462       }
1463       if (hasmm)
1464         emitir(IRTG(loadop, IRT_NIL), xref, 0);  /* Guard for nil value. */
1465       else if (xrefop == IR_HREF)
1466         emitir(IRTG(oldv == niltvg(J2G(J)) ? IR_EQ : IR_NE, IRT_PGC),
1467                xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1468       if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_newindex)) {
1469         lua_assert(hasmm);
1470         goto handlemm;
1471       }
1472       lua_assert(!hasmm);
1473       if (oldv == niltvg(J2G(J))) {  /* Need to insert a new key. */
1474         TRef key = ix->key;
1475         if (tref_isinteger(key))  /* NEWREF needs a TValue as a key. */
1476           key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
1477         xref = emitir(IRT(IR_NEWREF, IRT_PGC), ix->tab, key);
1478         keybarrier = 0;  /* NEWREF already takes care of the key barrier. */
1479 #ifdef LUAJIT_ENABLE_TABLE_BUMP
1480         if ((J->flags & JIT_F_OPT_SINK))  /* Avoid a separate flag. */
1481           rec_idx_bump(J, ix);
1482 #endif
1483       }
1484     } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) {
1485       /* Cannot derive that the previous value was non-nil, must do checks. */
1486       if (xrefop == IR_HREF)  /* Guard against store to niltv. */
1487         emitir(IRTG(IR_NE, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1488       if (ix->idxchain) {  /* Metamethod lookup required? */
1489         /* A check for NULL metatable is cheaper (hoistable) than a load. */
1490         if (!mt) {
1491           TRef mtref = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META);
1492           emitir(IRTG(IR_EQ, IRT_TAB), mtref, lj_ir_knull(J, IRT_TAB));
1493         } else {
1494           IRType t = itype2irt(oldv);
1495           emitir(IRTG(loadop, t), xref, 0);  /* Guard for non-nil value. */
1496         }
1497       }
1498     } else {
1499       keybarrier = 0;  /* Previous non-nil value kept the key alive. */
1500     }
1501     /* Convert int to number before storing. */
1502     if (!LJ_DUALNUM && tref_isinteger(ix->val))
1503       ix->val = emitir(IRTN(IR_CONV), ix->val, IRCONV_NUM_INT);
1504     emitir(IRT(loadop+IRDELTA_L2S, tref_type(ix->val)), xref, ix->val);
1505     if (keybarrier || tref_isgcv(ix->val))
1506       emitir(IRT(IR_TBAR, IRT_NIL), ix->tab, 0);
1507     /* Invalidate neg. metamethod cache for stores with certain string keys. */
1508     if (!nommstr(J, ix->key)) {
1509       TRef fref = emitir(IRT(IR_FREF, IRT_PGC), ix->tab, IRFL_TAB_NOMM);
1510       emitir(IRT(IR_FSTORE, IRT_U8), fref, lj_ir_kint(J, 0));
1511     }
1512     J->needsnap = 1;
1513     return 0;
1514   }
1515 }
1516 
1517 static void rec_tsetm(jit_State *J, BCReg ra, BCReg rn, int32_t i)
1518 {
1519   RecordIndex ix;
1520   cTValue *basev = J->L->base;
1521   GCtab *t = tabV(&basev[ra-1]);
1522   settabV(J->L, &ix.tabv, t);
1523   ix.tab = getslot(J, ra-1);
1524   ix.idxchain = 0;
1525 #ifdef LUAJIT_ENABLE_TABLE_BUMP
1526   if ((J->flags & JIT_F_OPT_SINK)) {
1527     if (t->asize < i+rn-ra)
1528       lj_tab_reasize(J->L, t, i+rn-ra);
1529     setnilV(&ix.keyv);
1530     rec_idx_bump(J, &ix);
1531   }
1532 #endif
1533   for (; ra < rn; i++, ra++) {
1534     setintV(&ix.keyv, i);
1535     ix.key = lj_ir_kint(J, i);
1536     copyTV(J->L, &ix.valv, &basev[ra]);
1537     ix.val = getslot(J, ra);
1538     lj_record_idx(J, &ix);
1539   }
1540 }
1541 
1542 /* -- Upvalue access ------------------------------------------------------ */
1543 
1544 /* Check whether upvalue is immutable and ok to constify. */
1545 static int rec_upvalue_constify(jit_State *J, GCupval *uvp)
1546 {
1547   if (uvp->immutable) {
1548     cTValue *o = uvval(uvp);
1549     /* Don't constify objects that may retain large amounts of memory. */
1550 #if LJ_HASFFI
1551     if (tviscdata(o)) {
1552       GCcdata *cd = cdataV(o);
1553       if (!cdataisv(cd) && !(cd->marked & LJ_GC_CDATA_FIN)) {
1554         CType *ct = ctype_raw(ctype_ctsG(J2G(J)), cd->ctypeid);
1555         if (!ctype_hassize(ct->info) || ct->size <= 16)
1556           return 1;
1557       }
1558       return 0;
1559     }
1560 #else
1561     UNUSED(J);
1562 #endif
1563     if (!(tvistab(o) || tvisudata(o) || tvisthread(o)))
1564       return 1;
1565   }
1566   return 0;
1567 }
1568 
1569 /* Record upvalue load/store. */
1570 static TRef rec_upvalue(jit_State *J, uint32_t uv, TRef val)
1571 {
1572   GCupval *uvp = &gcref(J->fn->l.uvptr[uv])->uv;
1573   TRef fn = getcurrf(J);
1574   IRRef uref;
1575   int needbarrier = 0;
1576   if (rec_upvalue_constify(J, uvp)) {  /* Try to constify immutable upvalue. */
1577     TRef tr, kfunc;
1578     lua_assert(val == 0);
1579     if (!tref_isk(fn)) {  /* Late specialization of current function. */
1580       if (J->pt->flags >= PROTO_CLC_POLY)
1581         goto noconstify;
1582       kfunc = lj_ir_kfunc(J, J->fn);
1583       emitir(IRTG(IR_EQ, IRT_FUNC), fn, kfunc);
1584 #if LJ_FR2
1585       J->base[-2] = kfunc;
1586 #else
1587       J->base[-1] = kfunc | TREF_FRAME;
1588 #endif
1589       fn = kfunc;
1590     }
1591     tr = lj_record_constify(J, uvval(uvp));
1592     if (tr)
1593       return tr;
1594   }
1595 noconstify:
1596   /* Note: this effectively limits LJ_MAX_UPVAL to 127. */
1597   uv = (uv << 8) | (hashrot(uvp->dhash, uvp->dhash + HASH_BIAS) & 0xff);
1598   if (!uvp->closed) {
1599     uref = tref_ref(emitir(IRTG(IR_UREFO, IRT_PGC), fn, uv));
1600     /* In current stack? */
1601     if (uvval(uvp) >= tvref(J->L->stack) &&
1602         uvval(uvp) < tvref(J->L->maxstack)) {
1603       int32_t slot = (int32_t)(uvval(uvp) - (J->L->base - J->baseslot));
1604       if (slot >= 0) {  /* Aliases an SSA slot? */
1605         emitir(IRTG(IR_EQ, IRT_PGC),
1606                REF_BASE,
1607                emitir(IRT(IR_ADD, IRT_PGC), uref,
1608                       lj_ir_kint(J, (slot - 1 - LJ_FR2) * -8)));
1609         slot -= (int32_t)J->baseslot;  /* Note: slot number may be negative! */
1610         if (val == 0) {
1611           return getslot(J, slot);
1612         } else {
1613           J->base[slot] = val;
1614           if (slot >= (int32_t)J->maxslot) J->maxslot = (BCReg)(slot+1);
1615           return 0;
1616         }
1617       }
1618     }
1619     emitir(IRTG(IR_UGT, IRT_PGC),
1620            emitir(IRT(IR_SUB, IRT_PGC), uref, REF_BASE),
1621            lj_ir_kint(J, (J->baseslot + J->maxslot) * 8));
1622   } else {
1623     needbarrier = 1;
1624     uref = tref_ref(emitir(IRTG(IR_UREFC, IRT_PGC), fn, uv));
1625   }
1626   if (val == 0) {  /* Upvalue load */
1627     IRType t = itype2irt(uvval(uvp));
1628     TRef res = emitir(IRTG(IR_ULOAD, t), uref, 0);
1629     if (irtype_ispri(t)) res = TREF_PRI(t);  /* Canonicalize primitive refs. */
1630     return res;
1631   } else {  /* Upvalue store. */
1632     /* Convert int to number before storing. */
1633     if (!LJ_DUALNUM && tref_isinteger(val))
1634       val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT);
1635     emitir(IRT(IR_USTORE, tref_type(val)), uref, val);
1636     if (needbarrier && tref_isgcv(val))
1637       emitir(IRT(IR_OBAR, IRT_NIL), uref, val);
1638     J->needsnap = 1;
1639     return 0;
1640   }
1641 }
1642 
1643 /* -- Record calls to Lua functions --------------------------------------- */
1644 
1645 /* Check unroll limits for calls. */
1646 static void check_call_unroll(jit_State *J, TraceNo lnk)
1647 {
1648   cTValue *frame = J->L->base - 1;
1649   void *pc = mref(frame_func(frame)->l.pc, void);
1650   int32_t depth = J->framedepth;
1651   int32_t count = 0;
1652   if ((J->pt->flags & PROTO_VARARG)) depth--;  /* Vararg frame still missing. */
1653   for (; depth > 0; depth--) {  /* Count frames with same prototype. */
1654     if (frame_iscont(frame)) depth--;
1655     frame = frame_prev(frame);
1656     if (mref(frame_func(frame)->l.pc, void) == pc)
1657       count++;
1658   }
1659   if (J->pc == J->startpc) {
1660     if (count + J->tailcalled > J->param[JIT_P_recunroll]) {
1661       J->pc++;
1662       if (J->framedepth + J->retdepth == 0)
1663         lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno);  /* Tail-rec. */
1664       else
1665         lj_record_stop(J, LJ_TRLINK_UPREC, J->cur.traceno);  /* Up-recursion. */
1666     }
1667   } else {
1668     if (count > J->param[JIT_P_callunroll]) {
1669       if (lnk) {  /* Possible tail- or up-recursion. */
1670         lj_trace_flush(J, lnk);  /* Flush trace that only returns. */
1671         /* Set a small, pseudo-random hotcount for a quick retry of JFUNC*. */
1672         hotcount_set(J2GG(J), J->pc+1, LJ_PRNG_BITS(J, 4));
1673       }
1674       lj_trace_err(J, LJ_TRERR_CUNROLL);
1675     }
1676   }
1677 }
1678 
1679 /* Record Lua function setup. */
1680 static void rec_func_setup(jit_State *J)
1681 {
1682   GCproto *pt = J->pt;
1683   BCReg s, numparams = pt->numparams;
1684   if ((pt->flags & PROTO_NOJIT))
1685     lj_trace_err(J, LJ_TRERR_CJITOFF);
1686   if (J->baseslot + pt->framesize >= LJ_MAX_JSLOTS)
1687     lj_trace_err(J, LJ_TRERR_STACKOV);
1688   /* Fill up missing parameters with nil. */
1689   for (s = J->maxslot; s < numparams; s++)
1690     J->base[s] = TREF_NIL;
1691   /* The remaining slots should never be read before they are written. */
1692   J->maxslot = numparams;
1693 }
1694 
1695 /* Record Lua vararg function setup. */
1696 static void rec_func_vararg(jit_State *J)
1697 {
1698   GCproto *pt = J->pt;
1699   BCReg s, fixargs, vframe = J->maxslot+1+LJ_FR2;
1700   lua_assert((pt->flags & PROTO_VARARG));
1701   if (J->baseslot + vframe + pt->framesize >= LJ_MAX_JSLOTS)
1702     lj_trace_err(J, LJ_TRERR_STACKOV);
1703   J->base[vframe-1-LJ_FR2] = J->base[-1-LJ_FR2];  /* Copy function up. */
1704 #if LJ_FR2
1705   J->base[vframe-1] = TREF_FRAME;
1706 #endif
1707   /* Copy fixarg slots up and set their original slots to nil. */
1708   fixargs = pt->numparams < J->maxslot ? pt->numparams : J->maxslot;
1709   for (s = 0; s < fixargs; s++) {
1710     J->base[vframe+s] = J->base[s];
1711     J->base[s] = TREF_NIL;
1712   }
1713   J->maxslot = fixargs;
1714   J->framedepth++;
1715   J->base += vframe;
1716   J->baseslot += vframe;
1717 }
1718 
1719 /* Record entry to a Lua function. */
1720 static void rec_func_lua(jit_State *J)
1721 {
1722   rec_func_setup(J);
1723   check_call_unroll(J, 0);
1724 }
1725 
1726 /* Record entry to an already compiled function. */
1727 static void rec_func_jit(jit_State *J, TraceNo lnk)
1728 {
1729   GCtrace *T;
1730   rec_func_setup(J);
1731   T = traceref(J, lnk);
1732   if (T->linktype == LJ_TRLINK_RETURN) {  /* Trace returns to interpreter? */
1733     check_call_unroll(J, lnk);
1734     /* Temporarily unpatch JFUNC* to continue recording across function. */
1735     J->patchins = *J->pc;
1736     J->patchpc = (BCIns *)J->pc;
1737     *J->patchpc = T->startins;
1738     return;
1739   }
1740   J->instunroll = 0;  /* Cannot continue across a compiled function. */
1741   if (J->pc == J->startpc && J->framedepth + J->retdepth == 0)
1742     lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno);  /* Extra tail-rec. */
1743   else
1744     lj_record_stop(J, LJ_TRLINK_ROOT, lnk);  /* Link to the function. */
1745 }
1746 
1747 /* -- Vararg handling ----------------------------------------------------- */
1748 
1749 /* Detect y = select(x, ...) idiom. */
1750 static int select_detect(jit_State *J)
1751 {
1752   BCIns ins = J->pc[1];
1753   if (bc_op(ins) == BC_CALLM && bc_b(ins) == 2 && bc_c(ins) == 1) {
1754     cTValue *func = &J->L->base[bc_a(ins)];
1755     if (tvisfunc(func) && funcV(func)->c.ffid == FF_select) {
1756       TRef kfunc = lj_ir_kfunc(J, funcV(func));
1757       emitir(IRTG(IR_EQ, IRT_FUNC), getslot(J, bc_a(ins)), kfunc);
1758       return 1;
1759     }
1760   }
1761   return 0;
1762 }
1763 
1764 /* Record vararg instruction. */
1765 static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults)
1766 {
1767   int32_t numparams = J->pt->numparams;
1768   ptrdiff_t nvararg = frame_delta(J->L->base-1) - numparams - 1 - LJ_FR2;
1769   lua_assert(frame_isvarg(J->L->base-1));
1770   if (LJ_FR2 && dst > J->maxslot)
1771     J->base[dst-1] = 0;  /* Prevent resurrection of unrelated slot. */
1772   if (J->framedepth > 0) {  /* Simple case: varargs defined on-trace. */
1773     ptrdiff_t i;
1774     if (nvararg < 0) nvararg = 0;
1775     if (nresults == -1) {
1776       nresults = nvararg;
1777       J->maxslot = dst + (BCReg)nvararg;
1778     } else if (dst + nresults > J->maxslot) {
1779       J->maxslot = dst + (BCReg)nresults;
1780     }
1781     for (i = 0; i < nresults; i++)
1782       J->base[dst+i] = i < nvararg ? getslot(J, i - nvararg - 1 - LJ_FR2) : TREF_NIL;
1783   } else {  /* Unknown number of varargs passed to trace. */
1784     TRef fr = emitir(IRTI(IR_SLOAD), LJ_FR2, IRSLOAD_READONLY|IRSLOAD_FRAME);
1785     int32_t frofs = 8*(1+LJ_FR2+numparams)+FRAME_VARG;
1786     if (nresults >= 0) {  /* Known fixed number of results. */
1787       ptrdiff_t i;
1788       if (nvararg > 0) {
1789         ptrdiff_t nload = nvararg >= nresults ? nresults : nvararg;
1790         TRef vbase;
1791         if (nvararg >= nresults)
1792           emitir(IRTGI(IR_GE), fr, lj_ir_kint(J, frofs+8*(int32_t)nresults));
1793         else
1794           emitir(IRTGI(IR_EQ), fr,
1795                  lj_ir_kint(J, (int32_t)frame_ftsz(J->L->base-1)));
1796         vbase = emitir(IRT(IR_SUB, IRT_IGC), REF_BASE, fr);
1797         vbase = emitir(IRT(IR_ADD, IRT_PGC), vbase, lj_ir_kint(J, frofs-8));
1798         for (i = 0; i < nload; i++) {
1799           IRType t = itype2irt(&J->L->base[i-1-LJ_FR2-nvararg]);
1800           TRef aref = emitir(IRT(IR_AREF, IRT_PGC),
1801                              vbase, lj_ir_kint(J, (int32_t)i));
1802           TRef tr = emitir(IRTG(IR_VLOAD, t), aref, 0);
1803           if (irtype_ispri(t)) tr = TREF_PRI(t);  /* Canonicalize primitives. */
1804           J->base[dst+i] = tr;
1805         }
1806       } else {
1807         emitir(IRTGI(IR_LE), fr, lj_ir_kint(J, frofs));
1808         nvararg = 0;
1809       }
1810       for (i = nvararg; i < nresults; i++)
1811         J->base[dst+i] = TREF_NIL;
1812       if (dst + (BCReg)nresults > J->maxslot)
1813         J->maxslot = dst + (BCReg)nresults;
1814     } else if (select_detect(J)) {  /* y = select(x, ...) */
1815       TRef tridx = J->base[dst-1];
1816       TRef tr = TREF_NIL;
1817       ptrdiff_t idx = lj_ffrecord_select_mode(J, tridx, &J->L->base[dst-1]);
1818       if (idx < 0) goto nyivarg;
1819       if (idx != 0 && !tref_isinteger(tridx))
1820         tridx = emitir(IRTGI(IR_CONV), tridx, IRCONV_INT_NUM|IRCONV_INDEX);
1821       if (idx != 0 && tref_isk(tridx)) {
1822         emitir(IRTGI(idx <= nvararg ? IR_GE : IR_LT),
1823                fr, lj_ir_kint(J, frofs+8*(int32_t)idx));
1824         frofs -= 8;  /* Bias for 1-based index. */
1825       } else if (idx <= nvararg) {  /* Compute size. */
1826         TRef tmp = emitir(IRTI(IR_ADD), fr, lj_ir_kint(J, -frofs));
1827         if (numparams)
1828           emitir(IRTGI(IR_GE), tmp, lj_ir_kint(J, 0));
1829         tr = emitir(IRTI(IR_BSHR), tmp, lj_ir_kint(J, 3));
1830         if (idx != 0) {
1831           tridx = emitir(IRTI(IR_ADD), tridx, lj_ir_kint(J, -1));
1832           rec_idx_abc(J, tr, tridx, (uint32_t)nvararg);
1833         }
1834       } else {
1835         TRef tmp = lj_ir_kint(J, frofs);
1836         if (idx != 0) {
1837           TRef tmp2 = emitir(IRTI(IR_BSHL), tridx, lj_ir_kint(J, 3));
1838           tmp = emitir(IRTI(IR_ADD), tmp2, tmp);
1839         } else {
1840           tr = lj_ir_kint(J, 0);
1841         }
1842         emitir(IRTGI(IR_LT), fr, tmp);
1843       }
1844       if (idx != 0 && idx <= nvararg) {
1845         IRType t;
1846         TRef aref, vbase = emitir(IRT(IR_SUB, IRT_IGC), REF_BASE, fr);
1847         vbase = emitir(IRT(IR_ADD, IRT_PGC), vbase,
1848                        lj_ir_kint(J, frofs-(8<<LJ_FR2)));
1849         t = itype2irt(&J->L->base[idx-2-LJ_FR2-nvararg]);
1850         aref = emitir(IRT(IR_AREF, IRT_PGC), vbase, tridx);
1851         tr = emitir(IRTG(IR_VLOAD, t), aref, 0);
1852         if (irtype_ispri(t)) tr = TREF_PRI(t);  /* Canonicalize primitives. */
1853       }
1854       J->base[dst-2-LJ_FR2] = tr;
1855       J->maxslot = dst-1-LJ_FR2;
1856       J->bcskip = 2;  /* Skip CALLM + select. */
1857     } else {
1858     nyivarg:
1859       setintV(&J->errinfo, BC_VARG);
1860       lj_trace_err_info(J, LJ_TRERR_NYIBC);
1861     }
1862   }
1863 }
1864 
1865 /* -- Record allocations -------------------------------------------------- */
1866 
1867 static TRef rec_tnew(jit_State *J, uint32_t ah)
1868 {
1869   uint32_t asize = ah & 0x7ff;
1870   uint32_t hbits = ah >> 11;
1871   TRef tr;
1872   if (asize == 0x7ff) asize = 0x801;
1873   tr = emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits);
1874 #ifdef LUAJIT_ENABLE_TABLE_BUMP
1875   J->rbchash[(tr & (RBCHASH_SLOTS-1))].ref = tref_ref(tr);
1876   setmref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pc, J->pc);
1877   setgcref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pt, obj2gco(J->pt));
1878 #endif
1879   return tr;
1880 }
1881 
1882 /* -- Concatenation ------------------------------------------------------- */
1883 
1884 static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot)
1885 {
1886   TRef *top = &J->base[topslot];
1887   TValue savetv[5];
1888   BCReg s;
1889   RecordIndex ix;
1890   lua_assert(baseslot < topslot);
1891   for (s = baseslot; s <= topslot; s++)
1892     (void)getslot(J, s);  /* Ensure all arguments have a reference. */
1893   if (tref_isnumber_str(top[0]) && tref_isnumber_str(top[-1])) {
1894     TRef tr, hdr, *trp, *xbase, *base = &J->base[baseslot];
1895     /* First convert numbers to strings. */
1896     for (trp = top; trp >= base; trp--) {
1897       if (tref_isnumber(*trp))
1898         *trp = emitir(IRT(IR_TOSTR, IRT_STR), *trp,
1899                       tref_isnum(*trp) ? IRTOSTR_NUM : IRTOSTR_INT);
1900       else if (!tref_isstr(*trp))
1901         break;
1902     }
1903     xbase = ++trp;
1904     tr = hdr = emitir(IRT(IR_BUFHDR, IRT_PGC),
1905                       lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET);
1906     do {
1907       tr = emitir(IRT(IR_BUFPUT, IRT_PGC), tr, *trp++);
1908     } while (trp <= top);
1909     tr = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr);
1910     J->maxslot = (BCReg)(xbase - J->base);
1911     if (xbase == base) return tr;  /* Return simple concatenation result. */
1912     /* Pass partial result. */
1913     topslot = J->maxslot--;
1914     *xbase = tr;
1915     top = xbase;
1916     setstrV(J->L, &ix.keyv, &J2G(J)->strempty);  /* Simulate string result. */
1917   } else {
1918     J->maxslot = topslot-1;
1919     copyTV(J->L, &ix.keyv, &J->L->base[topslot]);
1920   }
1921   copyTV(J->L, &ix.tabv, &J->L->base[topslot-1]);
1922   ix.tab = top[-1];
1923   ix.key = top[0];
1924   memcpy(savetv, &J->L->base[topslot-1], sizeof(savetv));  /* Save slots. */
1925   rec_mm_arith(J, &ix, MM_concat);  /* Call __concat metamethod. */
1926   memcpy(&J->L->base[topslot-1], savetv, sizeof(savetv));  /* Restore slots. */
1927   return 0;  /* No result yet. */
1928 }
1929 
1930 /* -- Record bytecode ops ------------------------------------------------- */
1931 
1932 /* Prepare for comparison. */
1933 static void rec_comp_prep(jit_State *J)
1934 {
1935   /* Prevent merging with snapshot #0 (GC exit) since we fixup the PC. */
1936   if (J->cur.nsnap == 1 && J->cur.snap[0].ref == J->cur.nins)
1937     emitir_raw(IRT(IR_NOP, IRT_NIL), 0, 0);
1938   lj_snap_add(J);
1939 }
1940 
1941 /* Fixup comparison. */
1942 static void rec_comp_fixup(jit_State *J, const BCIns *pc, int cond)
1943 {
1944   BCIns jmpins = pc[1];
1945   const BCIns *npc = pc + 2 + (cond ? bc_j(jmpins) : 0);
1946   SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
1947   /* Set PC to opposite target to avoid re-recording the comp. in side trace. */
1948 #if LJ_FR2
1949   SnapEntry *flink = &J->cur.snapmap[snap->mapofs + snap->nent];
1950   uint64_t pcbase;
1951   memcpy(&pcbase, flink, sizeof(uint64_t));
1952   pcbase = (pcbase & 0xff) | (u64ptr(npc) << 8);
1953   memcpy(flink, &pcbase, sizeof(uint64_t));
1954 #else
1955   J->cur.snapmap[snap->mapofs + snap->nent] = SNAP_MKPC(npc);
1956 #endif
1957   J->needsnap = 1;
1958   if (bc_a(jmpins) < J->maxslot) J->maxslot = bc_a(jmpins);
1959   lj_snap_shrink(J);  /* Shrink last snapshot if possible. */
1960 }
1961 
1962 /* Record the next bytecode instruction (_before_ it's executed). */
1963 void lj_record_ins(jit_State *J)
1964 {
1965   cTValue *lbase;
1966   RecordIndex ix;
1967   const BCIns *pc;
1968   BCIns ins;
1969   BCOp op;
1970   TRef ra, rb, rc;
1971 
1972   /* Perform post-processing action before recording the next instruction. */
1973   if (LJ_UNLIKELY(J->postproc != LJ_POST_NONE)) {
1974     switch (J->postproc) {
1975     case LJ_POST_FIXCOMP:  /* Fixup comparison. */
1976       pc = (const BCIns *)(uintptr_t)J2G(J)->tmptv.u64;
1977       rec_comp_fixup(J, pc, (!tvistruecond(&J2G(J)->tmptv2) ^ (bc_op(*pc)&1)));
1978       /* fallthrough */
1979     case LJ_POST_FIXGUARD:  /* Fixup and emit pending guard. */
1980     case LJ_POST_FIXGUARDSNAP:  /* Fixup and emit pending guard and snapshot. */
1981       if (!tvistruecond(&J2G(J)->tmptv2)) {
1982         J->fold.ins.o ^= 1;  /* Flip guard to opposite. */
1983         if (J->postproc == LJ_POST_FIXGUARDSNAP) {
1984           SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
1985           J->cur.snapmap[snap->mapofs+snap->nent-1]--;  /* False -> true. */
1986         }
1987       }
1988       lj_opt_fold(J);  /* Emit pending guard. */
1989       /* fallthrough */
1990     case LJ_POST_FIXBOOL:
1991       if (!tvistruecond(&J2G(J)->tmptv2)) {
1992         BCReg s;
1993         TValue *tv = J->L->base;
1994         for (s = 0; s < J->maxslot; s++)  /* Fixup stack slot (if any). */
1995           if (J->base[s] == TREF_TRUE && tvisfalse(&tv[s])) {
1996             J->base[s] = TREF_FALSE;
1997             break;
1998           }
1999       }
2000       break;
2001     case LJ_POST_FIXCONST:
2002       {
2003         BCReg s;
2004         TValue *tv = J->L->base;
2005         for (s = 0; s < J->maxslot; s++)  /* Constify stack slots (if any). */
2006           if (J->base[s] == TREF_NIL && !tvisnil(&tv[s]))
2007             J->base[s] = lj_record_constify(J, &tv[s]);
2008       }
2009       break;
2010     case LJ_POST_FFRETRY:  /* Suppress recording of retried fast function. */
2011       if (bc_op(*J->pc) >= BC__MAX)
2012         return;
2013       break;
2014     default: lua_assert(0); break;
2015     }
2016     J->postproc = LJ_POST_NONE;
2017   }
2018 
2019   /* Need snapshot before recording next bytecode (e.g. after a store). */
2020   if (J->needsnap) {
2021     J->needsnap = 0;
2022     lj_snap_purge(J);
2023     lj_snap_add(J);
2024     J->mergesnap = 1;
2025   }
2026 
2027   /* Skip some bytecodes. */
2028   if (LJ_UNLIKELY(J->bcskip > 0)) {
2029     J->bcskip--;
2030     return;
2031   }
2032 
2033   /* Record only closed loops for root traces. */
2034   pc = J->pc;
2035   if (J->framedepth == 0 &&
2036      (MSize)((char *)pc - (char *)J->bc_min) >= J->bc_extent)
2037     lj_trace_err(J, LJ_TRERR_LLEAVE);
2038 
2039 #ifdef LUA_USE_ASSERT
2040   rec_check_slots(J);
2041   rec_check_ir(J);
2042 #endif
2043 
2044 #if LJ_HASPROFILE
2045   rec_profile_ins(J, pc);
2046 #endif
2047 
2048   /* Keep a copy of the runtime values of var/num/str operands. */
2049 #define rav     (&ix.valv)
2050 #define rbv     (&ix.tabv)
2051 #define rcv     (&ix.keyv)
2052 
2053   lbase = J->L->base;
2054   ins = *pc;
2055   op = bc_op(ins);
2056   ra = bc_a(ins);
2057   ix.val = 0;
2058   switch (bcmode_a(op)) {
2059   case BCMvar:
2060     copyTV(J->L, rav, &lbase[ra]); ix.val = ra = getslot(J, ra); break;
2061   default: break;  /* Handled later. */
2062   }
2063   rb = bc_b(ins);
2064   rc = bc_c(ins);
2065   switch (bcmode_b(op)) {
2066   case BCMnone: rb = 0; rc = bc_d(ins); break;  /* Upgrade rc to 'rd'. */
2067   case BCMvar:
2068     copyTV(J->L, rbv, &lbase[rb]); ix.tab = rb = getslot(J, rb); break;
2069   default: break;  /* Handled later. */
2070   }
2071   switch (bcmode_c(op)) {
2072   case BCMvar:
2073     copyTV(J->L, rcv, &lbase[rc]); ix.key = rc = getslot(J, rc); break;
2074   case BCMpri: setpriV(rcv, ~rc); ix.key = rc = TREF_PRI(IRT_NIL+rc); break;
2075   case BCMnum: { cTValue *tv = proto_knumtv(J->pt, rc);
2076     copyTV(J->L, rcv, tv); ix.key = rc = tvisint(tv) ? lj_ir_kint(J, intV(tv)) :
2077     lj_ir_knumint(J, numV(tv)); } break;
2078   case BCMstr: { GCstr *s = gco2str(proto_kgc(J->pt, ~(ptrdiff_t)rc));
2079     setstrV(J->L, rcv, s); ix.key = rc = lj_ir_kstr(J, s); } break;
2080   default: break;  /* Handled later. */
2081   }
2082 
2083   switch (op) {
2084 
2085   /* -- Comparison ops ---------------------------------------------------- */
2086 
2087   case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2088 #if LJ_HASFFI
2089     if (tref_iscdata(ra) || tref_iscdata(rc)) {
2090       rec_mm_comp_cdata(J, &ix, op, ((int)op & 2) ? MM_le : MM_lt);
2091       break;
2092     }
2093 #endif
2094     /* Emit nothing for two numeric or string consts. */
2095     if (!(tref_isk2(ra,rc) && tref_isnumber_str(ra) && tref_isnumber_str(rc))) {
2096       IRType ta = tref_isinteger(ra) ? IRT_INT : tref_type(ra);
2097       IRType tc = tref_isinteger(rc) ? IRT_INT : tref_type(rc);
2098       int irop;
2099       if (ta != tc) {
2100         /* Widen mixed number/int comparisons to number/number comparison. */
2101         if (ta == IRT_INT && tc == IRT_NUM) {
2102           ra = emitir(IRTN(IR_CONV), ra, IRCONV_NUM_INT);
2103           ta = IRT_NUM;
2104         } else if (ta == IRT_NUM && tc == IRT_INT) {
2105           rc = emitir(IRTN(IR_CONV), rc, IRCONV_NUM_INT);
2106         } else if (LJ_52) {
2107           ta = IRT_NIL;  /* Force metamethod for different types. */
2108         } else if (!((ta == IRT_FALSE || ta == IRT_TRUE) &&
2109                      (tc == IRT_FALSE || tc == IRT_TRUE))) {
2110           break;  /* Interpreter will throw for two different types. */
2111         }
2112       }
2113       rec_comp_prep(J);
2114       irop = (int)op - (int)BC_ISLT + (int)IR_LT;
2115       if (ta == IRT_NUM) {
2116         if ((irop & 1)) irop ^= 4;  /* ISGE/ISGT are unordered. */
2117         if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop))
2118           irop ^= 5;
2119       } else if (ta == IRT_INT) {
2120         if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop))
2121           irop ^= 1;
2122       } else if (ta == IRT_STR) {
2123         if (!lj_ir_strcmp(strV(rav), strV(rcv), (IROp)irop)) irop ^= 1;
2124         ra = lj_ir_call(J, IRCALL_lj_str_cmp, ra, rc);
2125         rc = lj_ir_kint(J, 0);
2126         ta = IRT_INT;
2127       } else {
2128         rec_mm_comp(J, &ix, (int)op);
2129         break;
2130       }
2131       emitir(IRTG(irop, ta), ra, rc);
2132       rec_comp_fixup(J, J->pc, ((int)op ^ irop) & 1);
2133     }
2134     break;
2135 
2136   case BC_ISEQV: case BC_ISNEV:
2137   case BC_ISEQS: case BC_ISNES:
2138   case BC_ISEQN: case BC_ISNEN:
2139   case BC_ISEQP: case BC_ISNEP:
2140 #if LJ_HASFFI
2141     if (tref_iscdata(ra) || tref_iscdata(rc)) {
2142       rec_mm_comp_cdata(J, &ix, op, MM_eq);
2143       break;
2144     }
2145 #endif
2146     /* Emit nothing for two non-table, non-udata consts. */
2147     if (!(tref_isk2(ra, rc) && !(tref_istab(ra) || tref_isudata(ra)))) {
2148       int diff;
2149       rec_comp_prep(J);
2150       diff = lj_record_objcmp(J, ra, rc, rav, rcv);
2151       if (diff == 2 || !(tref_istab(ra) || tref_isudata(ra)))
2152         rec_comp_fixup(J, J->pc, ((int)op & 1) == !diff);
2153       else if (diff == 1)  /* Only check __eq if different, but same type. */
2154         rec_mm_equal(J, &ix, (int)op);
2155     }
2156     break;
2157 
2158   /* -- Unary test and copy ops ------------------------------------------- */
2159 
2160   case BC_ISTC: case BC_ISFC:
2161     if ((op & 1) == tref_istruecond(rc))
2162       rc = 0;  /* Don't store if condition is not true. */
2163     /* fallthrough */
2164   case BC_IST: case BC_ISF:  /* Type specialization suffices. */
2165     if (bc_a(pc[1]) < J->maxslot)
2166       J->maxslot = bc_a(pc[1]);  /* Shrink used slots. */
2167     break;
2168 
2169   case BC_ISTYPE: case BC_ISNUM:
2170     /* These coercions need to correspond with lj_meta_istype(). */
2171     if (LJ_DUALNUM && rc == ~LJ_TNUMX+1)
2172       ra = lj_opt_narrow_toint(J, ra);
2173     else if (rc == ~LJ_TNUMX+2)
2174       ra = lj_ir_tonum(J, ra);
2175     else if (rc == ~LJ_TSTR+1)
2176       ra = lj_ir_tostr(J, ra);
2177     /* else: type specialization suffices. */
2178     J->base[bc_a(ins)] = ra;
2179     break;
2180 
2181   /* -- Unary ops --------------------------------------------------------- */
2182 
2183   case BC_NOT:
2184     /* Type specialization already forces const result. */
2185     rc = tref_istruecond(rc) ? TREF_FALSE : TREF_TRUE;
2186     break;
2187 
2188   case BC_LEN:
2189     if (tref_isstr(rc))
2190       rc = emitir(IRTI(IR_FLOAD), rc, IRFL_STR_LEN);
2191     else if (!LJ_52 && tref_istab(rc))
2192       rc = lj_ir_call(J, IRCALL_lj_tab_len, rc);
2193     else
2194       rc = rec_mm_len(J, rc, rcv);
2195     break;
2196 
2197   /* -- Arithmetic ops ---------------------------------------------------- */
2198 
2199   case BC_UNM:
2200     if (tref_isnumber_str(rc)) {
2201       rc = lj_opt_narrow_unm(J, rc, rcv);
2202     } else {
2203       ix.tab = rc;
2204       copyTV(J->L, &ix.tabv, rcv);
2205       rc = rec_mm_arith(J, &ix, MM_unm);
2206     }
2207     break;
2208 
2209   case BC_ADDNV: case BC_SUBNV: case BC_MULNV: case BC_DIVNV: case BC_MODNV:
2210     /* Swap rb/rc and rbv/rcv. rav is temp. */
2211     ix.tab = rc; ix.key = rc = rb; rb = ix.tab;
2212     copyTV(J->L, rav, rbv);
2213     copyTV(J->L, rbv, rcv);
2214     copyTV(J->L, rcv, rav);
2215     if (op == BC_MODNV)
2216       goto recmod;
2217     /* fallthrough */
2218   case BC_ADDVN: case BC_SUBVN: case BC_MULVN: case BC_DIVVN:
2219   case BC_ADDVV: case BC_SUBVV: case BC_MULVV: case BC_DIVVV: {
2220     MMS mm = bcmode_mm(op);
2221     if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
2222       rc = lj_opt_narrow_arith(J, rb, rc, rbv, rcv,
2223                                (int)mm - (int)MM_add + (int)IR_ADD);
2224     else
2225       rc = rec_mm_arith(J, &ix, mm);
2226     break;
2227     }
2228 
2229   case BC_MODVN: case BC_MODVV:
2230   recmod:
2231     if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
2232       rc = lj_opt_narrow_mod(J, rb, rc, rbv, rcv);
2233     else
2234       rc = rec_mm_arith(J, &ix, MM_mod);
2235     break;
2236 
2237   case BC_POW:
2238     if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
2239       rc = lj_opt_narrow_pow(J, rb, rc, rbv, rcv);
2240     else
2241       rc = rec_mm_arith(J, &ix, MM_pow);
2242     break;
2243 
2244   /* -- Miscellaneous ops ------------------------------------------------- */
2245 
2246   case BC_CAT:
2247     rc = rec_cat(J, rb, rc);
2248     break;
2249 
2250   /* -- Constant and move ops --------------------------------------------- */
2251 
2252   case BC_MOV:
2253     /* Clear gap of method call to avoid resurrecting previous refs. */
2254     if (ra > J->maxslot) {
2255 #if LJ_FR2
2256       memset(J->base + J->maxslot, 0, (ra - J->maxslot) * sizeof(TRef));
2257 #else
2258       J->base[ra-1] = 0;
2259 #endif
2260     }
2261     break;
2262   case BC_KSTR: case BC_KNUM: case BC_KPRI:
2263     break;
2264   case BC_KSHORT:
2265     rc = lj_ir_kint(J, (int32_t)(int16_t)rc);
2266     break;
2267   case BC_KNIL:
2268     if (LJ_FR2 && ra > J->maxslot)
2269       J->base[ra-1] = 0;
2270     while (ra <= rc)
2271       J->base[ra++] = TREF_NIL;
2272     if (rc >= J->maxslot) J->maxslot = rc+1;
2273     break;
2274 #if LJ_HASFFI
2275   case BC_KCDATA:
2276     rc = lj_ir_kgc(J, proto_kgc(J->pt, ~(ptrdiff_t)rc), IRT_CDATA);
2277     break;
2278 #endif
2279 
2280   /* -- Upvalue and function ops ------------------------------------------ */
2281 
2282   case BC_UGET:
2283     rc = rec_upvalue(J, rc, 0);
2284     break;
2285   case BC_USETV: case BC_USETS: case BC_USETN: case BC_USETP:
2286     rec_upvalue(J, ra, rc);
2287     break;
2288 
2289   /* -- Table ops --------------------------------------------------------- */
2290 
2291   case BC_GGET: case BC_GSET:
2292     settabV(J->L, &ix.tabv, tabref(J->fn->l.env));
2293     ix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), getcurrf(J), IRFL_FUNC_ENV);
2294     ix.idxchain = LJ_MAX_IDXCHAIN;
2295     rc = lj_record_idx(J, &ix);
2296     break;
2297 
2298   case BC_TGETB: case BC_TSETB:
2299     setintV(&ix.keyv, (int32_t)rc);
2300     ix.key = lj_ir_kint(J, (int32_t)rc);
2301     /* fallthrough */
2302   case BC_TGETV: case BC_TGETS: case BC_TSETV: case BC_TSETS:
2303     ix.idxchain = LJ_MAX_IDXCHAIN;
2304     rc = lj_record_idx(J, &ix);
2305     break;
2306   case BC_TGETR: case BC_TSETR:
2307     ix.idxchain = 0;
2308     rc = lj_record_idx(J, &ix);
2309     break;
2310 
2311   case BC_TSETM:
2312     rec_tsetm(J, ra, (BCReg)(J->L->top - J->L->base), (int32_t)rcv->u32.lo);
2313     break;
2314 
2315   case BC_TNEW:
2316     rc = rec_tnew(J, rc);
2317     break;
2318   case BC_TDUP:
2319     rc = emitir(IRTG(IR_TDUP, IRT_TAB),
2320                 lj_ir_ktab(J, gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)rc))), 0);
2321 #ifdef LUAJIT_ENABLE_TABLE_BUMP
2322     J->rbchash[(rc & (RBCHASH_SLOTS-1))].ref = tref_ref(rc);
2323     setmref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pc, pc);
2324     setgcref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pt, obj2gco(J->pt));
2325 #endif
2326     break;
2327 
2328   /* -- Calls and vararg handling ----------------------------------------- */
2329 
2330   case BC_ITERC:
2331     J->base[ra] = getslot(J, ra-3);
2332     J->base[ra+1+LJ_FR2] = getslot(J, ra-2);
2333     J->base[ra+2+LJ_FR2] = getslot(J, ra-1);
2334     { /* Do the actual copy now because lj_record_call needs the values. */
2335       TValue *b = &J->L->base[ra];
2336       copyTV(J->L, b, b-3);
2337       copyTV(J->L, b+1+LJ_FR2, b-2);
2338       copyTV(J->L, b+2+LJ_FR2, b-1);
2339     }
2340     lj_record_call(J, ra, (ptrdiff_t)rc-1);
2341     break;
2342 
2343   /* L->top is set to L->base+ra+rc+NARGS-1+1. See lj_dispatch_ins(). */
2344   case BC_CALLM:
2345     rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2;
2346     /* fallthrough */
2347   case BC_CALL:
2348     lj_record_call(J, ra, (ptrdiff_t)rc-1);
2349     break;
2350 
2351   case BC_CALLMT:
2352     rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2;
2353     /* fallthrough */
2354   case BC_CALLT:
2355     lj_record_tailcall(J, ra, (ptrdiff_t)rc-1);
2356     break;
2357 
2358   case BC_VARG:
2359     rec_varg(J, ra, (ptrdiff_t)rb-1);
2360     break;
2361 
2362   /* -- Returns ----------------------------------------------------------- */
2363 
2364   case BC_RETM:
2365     /* L->top is set to L->base+ra+rc+NRESULTS-1, see lj_dispatch_ins(). */
2366     rc = (BCReg)(J->L->top - J->L->base) - ra + 1;
2367     /* fallthrough */
2368   case BC_RET: case BC_RET0: case BC_RET1:
2369 #if LJ_HASPROFILE
2370     rec_profile_ret(J);
2371 #endif
2372     lj_record_ret(J, ra, (ptrdiff_t)rc-1);
2373     break;
2374 
2375   /* -- Loops and branches ------------------------------------------------ */
2376 
2377   case BC_FORI:
2378     if (rec_for(J, pc, 0) != LOOPEV_LEAVE)
2379       J->loopref = J->cur.nins;
2380     break;
2381   case BC_JFORI:
2382     lua_assert(bc_op(pc[(ptrdiff_t)rc-BCBIAS_J]) == BC_JFORL);
2383     if (rec_for(J, pc, 0) != LOOPEV_LEAVE)  /* Link to existing loop. */
2384       lj_record_stop(J, LJ_TRLINK_ROOT, bc_d(pc[(ptrdiff_t)rc-BCBIAS_J]));
2385     /* Continue tracing if the loop is not entered. */
2386     break;
2387 
2388   case BC_FORL:
2389     rec_loop_interp(J, pc, rec_for(J, pc+((ptrdiff_t)rc-BCBIAS_J), 1));
2390     break;
2391   case BC_ITERL:
2392     rec_loop_interp(J, pc, rec_iterl(J, *pc));
2393     break;
2394   case BC_LOOP:
2395     rec_loop_interp(J, pc, rec_loop(J, ra));
2396     break;
2397 
2398   case BC_JFORL:
2399     rec_loop_jit(J, rc, rec_for(J, pc+bc_j(traceref(J, rc)->startins), 1));
2400     break;
2401   case BC_JITERL:
2402     rec_loop_jit(J, rc, rec_iterl(J, traceref(J, rc)->startins));
2403     break;
2404   case BC_JLOOP:
2405     rec_loop_jit(J, rc, rec_loop(J, ra));
2406     break;
2407 
2408   case BC_IFORL:
2409   case BC_IITERL:
2410   case BC_ILOOP:
2411   case BC_IFUNCF:
2412   case BC_IFUNCV:
2413     lj_trace_err(J, LJ_TRERR_BLACKL);
2414     break;
2415 
2416   case BC_JMP:
2417     if (ra < J->maxslot)
2418       J->maxslot = ra;  /* Shrink used slots. */
2419     break;
2420 
2421   /* -- Function headers -------------------------------------------------- */
2422 
2423   case BC_FUNCF:
2424     rec_func_lua(J);
2425     break;
2426   case BC_JFUNCF:
2427     rec_func_jit(J, rc);
2428     break;
2429 
2430   case BC_FUNCV:
2431     rec_func_vararg(J);
2432     rec_func_lua(J);
2433     break;
2434   case BC_JFUNCV:
2435     lua_assert(0);  /* Cannot happen. No hotcall counting for varag funcs. */
2436     break;
2437 
2438   case BC_FUNCC:
2439   case BC_FUNCCW:
2440     lj_ffrecord_func(J);
2441     break;
2442 
2443   default:
2444     if (op >= BC__MAX) {
2445       lj_ffrecord_func(J);
2446       break;
2447     }
2448     /* fallthrough */
2449   case BC_ITERN:
2450   case BC_ISNEXT:
2451   case BC_UCLO:
2452   case BC_FNEW:
2453     setintV(&J->errinfo, (int32_t)op);
2454     lj_trace_err_info(J, LJ_TRERR_NYIBC);
2455     break;
2456   }
2457 
2458   /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */
2459   if (bcmode_a(op) == BCMdst && rc) {
2460     J->base[ra] = rc;
2461     if (ra >= J->maxslot) {
2462 #if LJ_FR2
2463       if (ra > J->maxslot) J->base[ra-1] = 0;
2464 #endif
2465       J->maxslot = ra+1;
2466     }
2467   }
2468 
2469 #undef rav
2470 #undef rbv
2471 #undef rcv
2472 
2473   /* Limit the number of recorded IR instructions. */
2474   if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord])
2475     lj_trace_err(J, LJ_TRERR_TRACEOV);
2476 }
2477 
2478 /* -- Recording setup ----------------------------------------------------- */
2479 
2480 /* Setup recording for a root trace started by a hot loop. */
2481 static const BCIns *rec_setup_root(jit_State *J)
2482 {
2483   /* Determine the next PC and the bytecode range for the loop. */
2484   const BCIns *pcj, *pc = J->pc;
2485   BCIns ins = *pc;
2486   BCReg ra = bc_a(ins);
2487   switch (bc_op(ins)) {
2488   case BC_FORL:
2489     J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2490     pc += 1+bc_j(ins);
2491     J->bc_min = pc;
2492     break;
2493   case BC_ITERL:
2494     lua_assert(bc_op(pc[-1]) == BC_ITERC);
2495     J->maxslot = ra + bc_b(pc[-1]) - 1;
2496     J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2497     pc += 1+bc_j(ins);
2498     lua_assert(bc_op(pc[-1]) == BC_JMP);
2499     J->bc_min = pc;
2500     break;
2501   case BC_LOOP:
2502     /* Only check BC range for real loops, but not for "repeat until true". */
2503     pcj = pc + bc_j(ins);
2504     ins = *pcj;
2505     if (bc_op(ins) == BC_JMP && bc_j(ins) < 0) {
2506       J->bc_min = pcj+1 + bc_j(ins);
2507       J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2508     }
2509     J->maxslot = ra;
2510     pc++;
2511     break;
2512   case BC_RET:
2513   case BC_RET0:
2514   case BC_RET1:
2515     /* No bytecode range check for down-recursive root traces. */
2516     J->maxslot = ra + bc_d(ins) - 1;
2517     break;
2518   case BC_FUNCF:
2519     /* No bytecode range check for root traces started by a hot call. */
2520     J->maxslot = J->pt->numparams;
2521     pc++;
2522     break;
2523   case BC_CALLM:
2524   case BC_CALL:
2525   case BC_ITERC:
2526     /* No bytecode range check for stitched traces. */
2527     pc++;
2528     break;
2529   default:
2530     lua_assert(0);
2531     break;
2532   }
2533   return pc;
2534 }
2535 
2536 /* Setup for recording a new trace. */
2537 void lj_record_setup(jit_State *J)
2538 {
2539   uint32_t i;
2540 
2541   /* Initialize state related to current trace. */
2542   memset(J->slot, 0, sizeof(J->slot));
2543   memset(J->chain, 0, sizeof(J->chain));
2544 #ifdef LUAJIT_ENABLE_TABLE_BUMP
2545   memset(J->rbchash, 0, sizeof(J->rbchash));
2546 #endif
2547   memset(J->bpropcache, 0, sizeof(J->bpropcache));
2548   J->scev.idx = REF_NIL;
2549   setmref(J->scev.pc, NULL);
2550 
2551   J->baseslot = 1+LJ_FR2;  /* Invoking function is at base[-1-LJ_FR2]. */
2552   J->base = J->slot + J->baseslot;
2553   J->maxslot = 0;
2554   J->framedepth = 0;
2555   J->retdepth = 0;
2556 
2557   J->instunroll = J->param[JIT_P_instunroll];
2558   J->loopunroll = J->param[JIT_P_loopunroll];
2559   J->tailcalled = 0;
2560   J->loopref = 0;
2561 
2562   J->bc_min = NULL;  /* Means no limit. */
2563   J->bc_extent = ~(MSize)0;
2564 
2565   /* Emit instructions for fixed references. Also triggers initial IR alloc. */
2566   emitir_raw(IRT(IR_BASE, IRT_PGC), J->parent, J->exitno);
2567   for (i = 0; i <= 2; i++) {
2568     IRIns *ir = IR(REF_NIL-i);
2569     ir->i = 0;
2570     ir->t.irt = (uint8_t)(IRT_NIL+i);
2571     ir->o = IR_KPRI;
2572     ir->prev = 0;
2573   }
2574   J->cur.nk = REF_TRUE;
2575 
2576   J->startpc = J->pc;
2577   setmref(J->cur.startpc, J->pc);
2578   if (J->parent) {  /* Side trace. */
2579     GCtrace *T = traceref(J, J->parent);
2580     TraceNo root = T->root ? T->root : J->parent;
2581     J->cur.root = (uint16_t)root;
2582     J->cur.startins = BCINS_AD(BC_JMP, 0, 0);
2583     /* Check whether we could at least potentially form an extra loop. */
2584     if (J->exitno == 0 && T->snap[0].nent == 0) {
2585       /* We can narrow a FORL for some side traces, too. */
2586       if (J->pc > proto_bc(J->pt) && bc_op(J->pc[-1]) == BC_JFORI &&
2587           bc_d(J->pc[bc_j(J->pc[-1])-1]) == root) {
2588         lj_snap_add(J);
2589         rec_for_loop(J, J->pc-1, &J->scev, 1);
2590         goto sidecheck;
2591       }
2592     } else {
2593       J->startpc = NULL;  /* Prevent forming an extra loop. */
2594     }
2595     lj_snap_replay(J, T);
2596   sidecheck:
2597     if (traceref(J, J->cur.root)->nchild >= J->param[JIT_P_maxside] ||
2598         T->snap[J->exitno].count >= J->param[JIT_P_hotexit] +
2599                                     J->param[JIT_P_tryside]) {
2600       lj_record_stop(J, LJ_TRLINK_INTERP, 0);
2601     }
2602   } else {  /* Root trace. */
2603     J->cur.root = 0;
2604     J->cur.startins = *J->pc;
2605     J->pc = rec_setup_root(J);
2606     /* Note: the loop instruction itself is recorded at the end and not
2607     ** at the start! So snapshot #0 needs to point to the *next* instruction.
2608     */
2609     lj_snap_add(J);
2610     if (bc_op(J->cur.startins) == BC_FORL)
2611       rec_for_loop(J, J->pc-1, &J->scev, 1);
2612     else if (bc_op(J->cur.startins) == BC_ITERC)
2613       J->startpc = NULL;
2614     if (1 + J->pt->framesize >= LJ_MAX_JSLOTS)
2615       lj_trace_err(J, LJ_TRERR_STACKOV);
2616   }
2617 #if LJ_HASPROFILE
2618   J->prev_pt = NULL;
2619   J->prev_line = -1;
2620 #endif
2621 #ifdef LUAJIT_ENABLE_CHECKHOOK
2622   /* Regularly check for instruction/line hooks from compiled code and
2623   ** exit to the interpreter if the hooks are set.
2624   **
2625   ** This is a compile-time option and disabled by default, since the
2626   ** hook checks may be quite expensive in tight loops.
2627   **
2628   ** Note this is only useful if hooks are *not* set most of the time.
2629   ** Use this only if you want to *asynchronously* interrupt the execution.
2630   **
2631   ** You can set the instruction hook via lua_sethook() with a count of 1
2632   ** from a signal handler or another native thread. Please have a look
2633   ** at the first few functions in luajit.c for an example (Ctrl-C handler).
2634   */
2635   {
2636     TRef tr = emitir(IRT(IR_XLOAD, IRT_U8),
2637                      lj_ir_kptr(J, &J2G(J)->hookmask), IRXLOAD_VOLATILE);
2638     tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (LUA_MASKLINE|LUA_MASKCOUNT)));
2639     emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, 0));
2640   }
2641 #endif
2642 }
2643 
2644 #undef IR
2645 #undef emitir_raw
2646 #undef emitir
2647 
2648 #endif

/* [<][>][^][v][top][bottom][index][help] */