root/lj_trace.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. lj_trace_err
  2. lj_trace_err_info
  3. trace_findfree
  4. perftools_addtrace
  5. lj_trace_alloc
  6. trace_save
  7. lj_trace_free
  8. lj_trace_reenableproto
  9. trace_unpatch
  10. trace_flushroot
  11. lj_trace_flush
  12. lj_trace_flushproto
  13. lj_trace_flushall
  14. lj_trace_initstate
  15. lj_trace_freestate
  16. blacklist_pc
  17. penalty_pc
  18. trace_start
  19. trace_stop
  20. trace_downrec
  21. trace_abort
  22. trace_pendpatch
  23. trace_state
  24. lj_trace_ins
  25. lj_trace_hot
  26. trace_hotside
  27. lj_trace_stitch
  28. trace_exit_cp
  29. trace_exit_regs
  30. trace_exit_find
  31. lj_trace_exit

   1 /*
   2 ** Trace management.
   3 ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5 
   6 #define lj_trace_c
   7 #define LUA_CORE
   8 
   9 #include "lj_obj.h"
  10 
  11 #if LJ_HASJIT
  12 
  13 #include "lj_gc.h"
  14 #include "lj_err.h"
  15 #include "lj_debug.h"
  16 #include "lj_str.h"
  17 #include "lj_frame.h"
  18 #include "lj_state.h"
  19 #include "lj_bc.h"
  20 #include "lj_ir.h"
  21 #include "lj_jit.h"
  22 #include "lj_iropt.h"
  23 #include "lj_mcode.h"
  24 #include "lj_trace.h"
  25 #include "lj_snap.h"
  26 #include "lj_gdbjit.h"
  27 #include "lj_record.h"
  28 #include "lj_asm.h"
  29 #include "lj_dispatch.h"
  30 #include "lj_vm.h"
  31 #include "lj_vmevent.h"
  32 #include "lj_target.h"
  33 
  34 /* -- Error handling ------------------------------------------------------ */
  35 
  36 /* Synchronous abort with error message. */
  37 void lj_trace_err(jit_State *J, TraceError e)
  38 {
  39   setnilV(&J->errinfo);  /* No error info. */
  40   setintV(J->L->top++, (int32_t)e);
  41   lj_err_throw(J->L, LUA_ERRRUN);
  42 }
  43 
  44 /* Synchronous abort with error message and error info. */
  45 void lj_trace_err_info(jit_State *J, TraceError e)
  46 {
  47   setintV(J->L->top++, (int32_t)e);
  48   lj_err_throw(J->L, LUA_ERRRUN);
  49 }
  50 
  51 /* -- Trace management ---------------------------------------------------- */
  52 
  53 /* The current trace is first assembled in J->cur. The variable length
  54 ** arrays point to shared, growable buffers (J->irbuf etc.). When trace
  55 ** recording ends successfully, the current trace and its data structures
  56 ** are copied to a new (compact) GCtrace object.
  57 */
  58 
  59 /* Find a free trace number. */
  60 static TraceNo trace_findfree(jit_State *J)
  61 {
  62   MSize osz, lim;
  63   if (J->freetrace == 0)
  64     J->freetrace = 1;
  65   for (; J->freetrace < J->sizetrace; J->freetrace++)
  66     if (traceref(J, J->freetrace) == NULL)
  67       return J->freetrace++;
  68   /* Need to grow trace array. */
  69   lim = (MSize)J->param[JIT_P_maxtrace] + 1;
  70   if (lim < 2) lim = 2; else if (lim > 65535) lim = 65535;
  71   osz = J->sizetrace;
  72   if (osz >= lim)
  73     return 0;  /* Too many traces. */
  74   lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef);
  75   for (; osz < J->sizetrace; osz++)
  76     setgcrefnull(J->trace[osz]);
  77   return J->freetrace;
  78 }
  79 
  80 #define TRACE_APPENDVEC(field, szfield, tp) \
  81   T->field = (tp *)p; \
  82   memcpy(p, J->cur.field, J->cur.szfield*sizeof(tp)); \
  83   p += J->cur.szfield*sizeof(tp);
  84 
  85 #ifdef LUAJIT_USE_PERFTOOLS
  86 /*
  87 ** Create symbol table of JIT-compiled code. For use with Linux perf tools.
  88 ** Example usage:
  89 **   perf record -f -e cycles luajit test.lua
  90 **   perf report -s symbol
  91 **   rm perf.data /tmp/perf-*.map
  92 */
  93 #include <stdio.h>
  94 #include <unistd.h>
  95 
  96 static void perftools_addtrace(GCtrace *T)
  97 {
  98   static FILE *fp;
  99   GCproto *pt = &gcref(T->startpt)->pt;
 100   const BCIns *startpc = mref(T->startpc, const BCIns);
 101   const char *name = proto_chunknamestr(pt);
 102   BCLine lineno;
 103   if (name[0] == '@' || name[0] == '=')
 104     name++;
 105   else
 106     name = "(string)";
 107   lua_assert(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc);
 108   lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
 109   if (!fp) {
 110     char fname[40];
 111     sprintf(fname, "/tmp/perf-%d.map", getpid());
 112     if (!(fp = fopen(fname, "w"))) return;
 113     setlinebuf(fp);
 114   }
 115   fprintf(fp, "%lx %x TRACE_%d::%s:%u\n",
 116           (long)T->mcode, T->szmcode, T->traceno, name, lineno);
 117 }
 118 #endif
 119 
 120 /* Allocate space for copy of T. */
 121 GCtrace * LJ_FASTCALL lj_trace_alloc(lua_State *L, GCtrace *T)
 122 {
 123   size_t sztr = ((sizeof(GCtrace)+7)&~7);
 124   size_t szins = (T->nins-T->nk)*sizeof(IRIns);
 125   size_t sz = sztr + szins +
 126               T->nsnap*sizeof(SnapShot) +
 127               T->nsnapmap*sizeof(SnapEntry);
 128   GCtrace *T2 = lj_mem_newt(L, (MSize)sz, GCtrace);
 129   char *p = (char *)T2 + sztr;
 130   T2->gct = ~LJ_TTRACE;
 131   T2->marked = 0;
 132   T2->traceno = 0;
 133   T2->ir = (IRIns *)p - T->nk;
 134   T2->nins = T->nins;
 135   T2->nk = T->nk;
 136   T2->nsnap = T->nsnap;
 137   T2->nsnapmap = T->nsnapmap;
 138   memcpy(p, T->ir + T->nk, szins);
 139   return T2;
 140 }
 141 
 142 /* Save current trace by copying and compacting it. */
 143 static void trace_save(jit_State *J, GCtrace *T)
 144 {
 145   size_t sztr = ((sizeof(GCtrace)+7)&~7);
 146   size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns);
 147   char *p = (char *)T + sztr;
 148   memcpy(T, &J->cur, sizeof(GCtrace));
 149   setgcrefr(T->nextgc, J2G(J)->gc.root);
 150   setgcrefp(J2G(J)->gc.root, T);
 151   newwhite(J2G(J), T);
 152   T->gct = ~LJ_TTRACE;
 153   T->ir = (IRIns *)p - J->cur.nk;  /* The IR has already been copied above. */
 154   p += szins;
 155   TRACE_APPENDVEC(snap, nsnap, SnapShot)
 156   TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry)
 157   J->cur.traceno = 0;
 158   J->curfinal = NULL;
 159   setgcrefp(J->trace[T->traceno], T);
 160   lj_gc_barriertrace(J2G(J), T->traceno);
 161   lj_gdbjit_addtrace(J, T);
 162 #ifdef LUAJIT_USE_PERFTOOLS
 163   perftools_addtrace(T);
 164 #endif
 165 }
 166 
 167 void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T)
 168 {
 169   jit_State *J = G2J(g);
 170   if (T->traceno) {
 171     lj_gdbjit_deltrace(J, T);
 172     if (T->traceno < J->freetrace)
 173       J->freetrace = T->traceno;
 174     setgcrefnull(J->trace[T->traceno]);
 175   }
 176   lj_mem_free(g, T,
 177     ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
 178     T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
 179 }
 180 
 181 /* Re-enable compiling a prototype by unpatching any modified bytecode. */
 182 void lj_trace_reenableproto(GCproto *pt)
 183 {
 184   if ((pt->flags & PROTO_ILOOP)) {
 185     BCIns *bc = proto_bc(pt);
 186     BCPos i, sizebc = pt->sizebc;;
 187     pt->flags &= ~PROTO_ILOOP;
 188     if (bc_op(bc[0]) == BC_IFUNCF)
 189       setbc_op(&bc[0], BC_FUNCF);
 190     for (i = 1; i < sizebc; i++) {
 191       BCOp op = bc_op(bc[i]);
 192       if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP)
 193         setbc_op(&bc[i], (int)op+(int)BC_LOOP-(int)BC_ILOOP);
 194     }
 195   }
 196 }
 197 
 198 /* Unpatch the bytecode modified by a root trace. */
 199 static void trace_unpatch(jit_State *J, GCtrace *T)
 200 {
 201   BCOp op = bc_op(T->startins);
 202   BCIns *pc = mref(T->startpc, BCIns);
 203   UNUSED(J);
 204   if (op == BC_JMP)
 205     return;  /* No need to unpatch branches in parent traces (yet). */
 206   switch (bc_op(*pc)) {
 207   case BC_JFORL:
 208     lua_assert(traceref(J, bc_d(*pc)) == T);
 209     *pc = T->startins;
 210     pc += bc_j(T->startins);
 211     lua_assert(bc_op(*pc) == BC_JFORI);
 212     setbc_op(pc, BC_FORI);
 213     break;
 214   case BC_JITERL:
 215   case BC_JLOOP:
 216     lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op));
 217     *pc = T->startins;
 218     break;
 219   case BC_JMP:
 220     lua_assert(op == BC_ITERL);
 221     pc += bc_j(*pc)+2;
 222     if (bc_op(*pc) == BC_JITERL) {
 223       lua_assert(traceref(J, bc_d(*pc)) == T);
 224       *pc = T->startins;
 225     }
 226     break;
 227   case BC_JFUNCF:
 228     lua_assert(op == BC_FUNCF);
 229     *pc = T->startins;
 230     break;
 231   default:  /* Already unpatched. */
 232     break;
 233   }
 234 }
 235 
 236 /* Flush a root trace. */
 237 static void trace_flushroot(jit_State *J, GCtrace *T)
 238 {
 239   GCproto *pt = &gcref(T->startpt)->pt;
 240   lua_assert(T->root == 0 && pt != NULL);
 241   /* First unpatch any modified bytecode. */
 242   trace_unpatch(J, T);
 243   /* Unlink root trace from chain anchored in prototype. */
 244   if (pt->trace == T->traceno) {  /* Trace is first in chain. Easy. */
 245     pt->trace = T->nextroot;
 246   } else if (pt->trace) {  /* Otherwise search in chain of root traces. */
 247     GCtrace *T2 = traceref(J, pt->trace);
 248     if (T2) {
 249       for (; T2->nextroot; T2 = traceref(J, T2->nextroot))
 250         if (T2->nextroot == T->traceno) {
 251           T2->nextroot = T->nextroot;  /* Unlink from chain. */
 252           break;
 253         }
 254     }
 255   }
 256 }
 257 
 258 /* Flush a trace. Only root traces are considered. */
 259 void lj_trace_flush(jit_State *J, TraceNo traceno)
 260 {
 261   if (traceno > 0 && traceno < J->sizetrace) {
 262     GCtrace *T = traceref(J, traceno);
 263     if (T && T->root == 0)
 264       trace_flushroot(J, T);
 265   }
 266 }
 267 
 268 /* Flush all traces associated with a prototype. */
 269 void lj_trace_flushproto(global_State *g, GCproto *pt)
 270 {
 271   while (pt->trace != 0)
 272     trace_flushroot(G2J(g), traceref(G2J(g), pt->trace));
 273 }
 274 
 275 /* Flush all traces. */
 276 int lj_trace_flushall(lua_State *L)
 277 {
 278   jit_State *J = L2J(L);
 279   ptrdiff_t i;
 280   if ((J2G(J)->hookmask & HOOK_GC))
 281     return 1;
 282   for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) {
 283     GCtrace *T = traceref(J, i);
 284     if (T) {
 285       if (T->root == 0)
 286         trace_flushroot(J, T);
 287       lj_gdbjit_deltrace(J, T);
 288       T->traceno = T->link = 0;  /* Blacklist the link for cont_stitch. */
 289       setgcrefnull(J->trace[i]);
 290     }
 291   }
 292   J->cur.traceno = 0;
 293   J->freetrace = 0;
 294   /* Clear penalty cache. */
 295   memset(J->penalty, 0, sizeof(J->penalty));
 296   /* Free the whole machine code and invalidate all exit stub groups. */
 297   lj_mcode_free(J);
 298   memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup));
 299   lj_vmevent_send(L, TRACE,
 300     setstrV(L, L->top++, lj_str_newlit(L, "flush"));
 301   );
 302   return 0;
 303 }
 304 
 305 /* Initialize JIT compiler state. */
 306 void lj_trace_initstate(global_State *g)
 307 {
 308   jit_State *J = G2J(g);
 309   TValue *tv;
 310 
 311   /* Initialize aligned SIMD constants. */
 312   tv = LJ_KSIMD(J, LJ_KSIMD_ABS);
 313   tv[0].u64 = U64x(7fffffff,ffffffff);
 314   tv[1].u64 = U64x(7fffffff,ffffffff);
 315   tv = LJ_KSIMD(J, LJ_KSIMD_NEG);
 316   tv[0].u64 = U64x(80000000,00000000);
 317   tv[1].u64 = U64x(80000000,00000000);
 318 
 319   /* Initialize 32/64 bit constants. */
 320 #if LJ_TARGET_X86ORX64
 321   J->k64[LJ_K64_TOBIT].u64 = U64x(43380000,00000000);
 322 #if LJ_32
 323   J->k64[LJ_K64_M2P64_31].u64 = U64x(c1e00000,00000000);
 324 #endif
 325   J->k64[LJ_K64_2P64].u64 = U64x(43f00000,00000000);
 326   J->k32[LJ_K32_M2P64_31] = LJ_64 ? 0xdf800000 : 0xcf000000;
 327 #endif
 328 #if LJ_TARGET_X86ORX64 || LJ_TARGET_MIPS64
 329   J->k64[LJ_K64_M2P64].u64 = U64x(c3f00000,00000000);
 330 #endif
 331 #if LJ_TARGET_PPC
 332   J->k32[LJ_K32_2P52_2P31] = 0x59800004;
 333   J->k32[LJ_K32_2P52] = 0x59800000;
 334 #endif
 335 #if LJ_TARGET_PPC || LJ_TARGET_MIPS
 336   J->k32[LJ_K32_2P31] = 0x4f000000;
 337 #endif
 338 #if LJ_TARGET_MIPS
 339   J->k64[LJ_K64_2P31].u64 = U64x(41e00000,00000000);
 340 #if LJ_64
 341   J->k64[LJ_K64_2P63].u64 = U64x(43e00000,00000000);
 342   J->k32[LJ_K32_2P63] = 0x5f000000;
 343   J->k32[LJ_K32_M2P64] = 0xdf800000;
 344 #endif
 345 #endif
 346 }
 347 
 348 /* Free everything associated with the JIT compiler state. */
 349 void lj_trace_freestate(global_State *g)
 350 {
 351   jit_State *J = G2J(g);
 352 #ifdef LUA_USE_ASSERT
 353   {  /* This assumes all traces have already been freed. */
 354     ptrdiff_t i;
 355     for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
 356       lua_assert(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL);
 357   }
 358 #endif
 359   lj_mcode_free(J);
 360   lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
 361   lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
 362   lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
 363   lj_mem_freevec(g, J->trace, J->sizetrace, GCRef);
 364 }
 365 
 366 /* -- Penalties and blacklisting ------------------------------------------ */
 367 
 368 /* Blacklist a bytecode instruction. */
 369 static void blacklist_pc(GCproto *pt, BCIns *pc)
 370 {
 371   setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP);
 372   pt->flags |= PROTO_ILOOP;
 373 }
 374 
 375 /* Penalize a bytecode instruction. */
 376 static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e)
 377 {
 378   uint32_t i, val = PENALTY_MIN;
 379   for (i = 0; i < PENALTY_SLOTS; i++)
 380     if (mref(J->penalty[i].pc, const BCIns) == pc) {  /* Cache slot found? */
 381       /* First try to bump its hotcount several times. */
 382       val = ((uint32_t)J->penalty[i].val << 1) +
 383             LJ_PRNG_BITS(J, PENALTY_RNDBITS);
 384       if (val > PENALTY_MAX) {
 385         blacklist_pc(pt, pc);  /* Blacklist it, if that didn't help. */
 386         return;
 387       }
 388       goto setpenalty;
 389     }
 390   /* Assign a new penalty cache slot. */
 391   i = J->penaltyslot;
 392   J->penaltyslot = (J->penaltyslot + 1) & (PENALTY_SLOTS-1);
 393   setmref(J->penalty[i].pc, pc);
 394 setpenalty:
 395   J->penalty[i].val = (uint16_t)val;
 396   J->penalty[i].reason = e;
 397   hotcount_set(J2GG(J), pc+1, val);
 398 }
 399 
 400 /* -- Trace compiler state machine ---------------------------------------- */
 401 
 402 /* Start tracing. */
 403 static void trace_start(jit_State *J)
 404 {
 405   lua_State *L;
 406   TraceNo traceno;
 407 
 408   if ((J->pt->flags & PROTO_NOJIT)) {  /* JIT disabled for this proto? */
 409     if (J->parent == 0 && J->exitno == 0) {
 410       /* Lazy bytecode patching to disable hotcount events. */
 411       lua_assert(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL ||
 412                  bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF);
 413       setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP);
 414       J->pt->flags |= PROTO_ILOOP;
 415     }
 416     J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
 417     return;
 418   }
 419 
 420   /* Get a new trace number. */
 421   traceno = trace_findfree(J);
 422   if (LJ_UNLIKELY(traceno == 0)) {  /* No free trace? */
 423     lua_assert((J2G(J)->hookmask & HOOK_GC) == 0);
 424     lj_trace_flushall(J->L);
 425     J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
 426     return;
 427   }
 428   setgcrefp(J->trace[traceno], &J->cur);
 429 
 430   /* Setup enough of the current trace to be able to send the vmevent. */
 431   memset(&J->cur, 0, sizeof(GCtrace));
 432   J->cur.traceno = traceno;
 433   J->cur.nins = J->cur.nk = REF_BASE;
 434   J->cur.ir = J->irbuf;
 435   J->cur.snap = J->snapbuf;
 436   J->cur.snapmap = J->snapmapbuf;
 437   J->mergesnap = 0;
 438   J->needsnap = 0;
 439   J->bcskip = 0;
 440   J->guardemit.irt = 0;
 441   J->postproc = LJ_POST_NONE;
 442   lj_resetsplit(J);
 443   J->retryrec = 0;
 444   J->ktrace = 0;
 445   setgcref(J->cur.startpt, obj2gco(J->pt));
 446 
 447   L = J->L;
 448   lj_vmevent_send(L, TRACE,
 449     setstrV(L, L->top++, lj_str_newlit(L, "start"));
 450     setintV(L->top++, traceno);
 451     setfuncV(L, L->top++, J->fn);
 452     setintV(L->top++, proto_bcpos(J->pt, J->pc));
 453     if (J->parent) {
 454       setintV(L->top++, J->parent);
 455       setintV(L->top++, J->exitno);
 456     } else {
 457       BCOp op = bc_op(*J->pc);
 458       if (op == BC_CALLM || op == BC_CALL || op == BC_ITERC) {
 459         setintV(L->top++, J->exitno);  /* Parent of stitched trace. */
 460         setintV(L->top++, -1);
 461       }
 462     }
 463   );
 464   lj_record_setup(J);
 465 }
 466 
 467 /* Stop tracing. */
 468 static void trace_stop(jit_State *J)
 469 {
 470   BCIns *pc = mref(J->cur.startpc, BCIns);
 471   BCOp op = bc_op(J->cur.startins);
 472   GCproto *pt = &gcref(J->cur.startpt)->pt;
 473   TraceNo traceno = J->cur.traceno;
 474   GCtrace *T = J->curfinal;
 475   lua_State *L;
 476 
 477   switch (op) {
 478   case BC_FORL:
 479     setbc_op(pc+bc_j(J->cur.startins), BC_JFORI);  /* Patch FORI, too. */
 480     /* fallthrough */
 481   case BC_LOOP:
 482   case BC_ITERL:
 483   case BC_FUNCF:
 484     /* Patch bytecode of starting instruction in root trace. */
 485     setbc_op(pc, (int)op+(int)BC_JLOOP-(int)BC_LOOP);
 486     setbc_d(pc, traceno);
 487   addroot:
 488     /* Add to root trace chain in prototype. */
 489     J->cur.nextroot = pt->trace;
 490     pt->trace = (TraceNo1)traceno;
 491     break;
 492   case BC_RET:
 493   case BC_RET0:
 494   case BC_RET1:
 495     *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, traceno);
 496     goto addroot;
 497   case BC_JMP:
 498     /* Patch exit branch in parent to side trace entry. */
 499     lua_assert(J->parent != 0 && J->cur.root != 0);
 500     lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
 501     /* Avoid compiling a side trace twice (stack resizing uses parent exit). */
 502     traceref(J, J->parent)->snap[J->exitno].count = SNAPCOUNT_DONE;
 503     /* Add to side trace chain in root trace. */
 504     {
 505       GCtrace *root = traceref(J, J->cur.root);
 506       root->nchild++;
 507       J->cur.nextside = root->nextside;
 508       root->nextside = (TraceNo1)traceno;
 509     }
 510     break;
 511   case BC_CALLM:
 512   case BC_CALL:
 513   case BC_ITERC:
 514     /* Trace stitching: patch link of previous trace. */
 515     traceref(J, J->exitno)->link = traceno;
 516     break;
 517   default:
 518     lua_assert(0);
 519     break;
 520   }
 521 
 522   /* Commit new mcode only after all patching is done. */
 523   lj_mcode_commit(J, J->cur.mcode);
 524   J->postproc = LJ_POST_NONE;
 525   trace_save(J, T);
 526 
 527   L = J->L;
 528   lj_vmevent_send(L, TRACE,
 529     setstrV(L, L->top++, lj_str_newlit(L, "stop"));
 530     setintV(L->top++, traceno);
 531     setfuncV(L, L->top++, J->fn);
 532   );
 533 }
 534 
 535 /* Start a new root trace for down-recursion. */
 536 static int trace_downrec(jit_State *J)
 537 {
 538   /* Restart recording at the return instruction. */
 539   lua_assert(J->pt != NULL);
 540   lua_assert(bc_isret(bc_op(*J->pc)));
 541   if (bc_op(*J->pc) == BC_RETM)
 542     return 0;  /* NYI: down-recursion with RETM. */
 543   J->parent = 0;
 544   J->exitno = 0;
 545   J->state = LJ_TRACE_RECORD;
 546   trace_start(J);
 547   return 1;
 548 }
 549 
 550 /* Abort tracing. */
 551 static int trace_abort(jit_State *J)
 552 {
 553   lua_State *L = J->L;
 554   TraceError e = LJ_TRERR_RECERR;
 555   TraceNo traceno;
 556 
 557   J->postproc = LJ_POST_NONE;
 558   lj_mcode_abort(J);
 559   if (J->curfinal) {
 560     lj_trace_free(J2G(J), J->curfinal);
 561     J->curfinal = NULL;
 562   }
 563   if (tvisnumber(L->top-1))
 564     e = (TraceError)numberVint(L->top-1);
 565   if (e == LJ_TRERR_MCODELM) {
 566     L->top--;  /* Remove error object */
 567     J->state = LJ_TRACE_ASM;
 568     return 1;  /* Retry ASM with new MCode area. */
 569   }
 570   /* Penalize or blacklist starting bytecode instruction. */
 571   if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) {
 572     if (J->exitno == 0) {
 573       BCIns *startpc = mref(J->cur.startpc, BCIns);
 574       if (e == LJ_TRERR_RETRY)
 575         hotcount_set(J2GG(J), startpc+1, 1);  /* Immediate retry. */
 576       else
 577         penalty_pc(J, &gcref(J->cur.startpt)->pt, startpc, e);
 578     } else {
 579       traceref(J, J->exitno)->link = J->exitno;  /* Self-link is blacklisted. */
 580     }
 581   }
 582 
 583   /* Is there anything to abort? */
 584   traceno = J->cur.traceno;
 585   if (traceno) {
 586     ptrdiff_t errobj = savestack(L, L->top-1);  /* Stack may be resized. */
 587     J->cur.link = 0;
 588     J->cur.linktype = LJ_TRLINK_NONE;
 589     lj_vmevent_send(L, TRACE,
 590       TValue *frame;
 591       const BCIns *pc;
 592       GCfunc *fn;
 593       setstrV(L, L->top++, lj_str_newlit(L, "abort"));
 594       setintV(L->top++, traceno);
 595       /* Find original Lua function call to generate a better error message. */
 596       frame = J->L->base-1;
 597       pc = J->pc;
 598       while (!isluafunc(frame_func(frame))) {
 599         pc = (frame_iscont(frame) ? frame_contpc(frame) : frame_pc(frame)) - 1;
 600         frame = frame_prev(frame);
 601       }
 602       fn = frame_func(frame);
 603       setfuncV(L, L->top++, fn);
 604       setintV(L->top++, proto_bcpos(funcproto(fn), pc));
 605       copyTV(L, L->top++, restorestack(L, errobj));
 606       copyTV(L, L->top++, &J->errinfo);
 607     );
 608     /* Drop aborted trace after the vmevent (which may still access it). */
 609     setgcrefnull(J->trace[traceno]);
 610     if (traceno < J->freetrace)
 611       J->freetrace = traceno;
 612     J->cur.traceno = 0;
 613   }
 614   L->top--;  /* Remove error object */
 615   if (e == LJ_TRERR_DOWNREC)
 616     return trace_downrec(J);
 617   else if (e == LJ_TRERR_MCODEAL)
 618     lj_trace_flushall(L);
 619   return 0;
 620 }
 621 
 622 /* Perform pending re-patch of a bytecode instruction. */
 623 static LJ_AINLINE void trace_pendpatch(jit_State *J, int force)
 624 {
 625   if (LJ_UNLIKELY(J->patchpc)) {
 626     if (force || J->bcskip == 0) {
 627       *J->patchpc = J->patchins;
 628       J->patchpc = NULL;
 629     } else {
 630       J->bcskip = 0;
 631     }
 632   }
 633 }
 634 
 635 /* State machine for the trace compiler. Protected callback. */
 636 static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud)
 637 {
 638   jit_State *J = (jit_State *)ud;
 639   UNUSED(dummy);
 640   do {
 641   retry:
 642     switch (J->state) {
 643     case LJ_TRACE_START:
 644       J->state = LJ_TRACE_RECORD;  /* trace_start() may change state. */
 645       trace_start(J);
 646       lj_dispatch_update(J2G(J));
 647       break;
 648 
 649     case LJ_TRACE_RECORD:
 650       trace_pendpatch(J, 0);
 651       setvmstate(J2G(J), RECORD);
 652       lj_vmevent_send_(L, RECORD,
 653         /* Save/restore tmptv state for trace recorder. */
 654         TValue savetv = J2G(J)->tmptv;
 655         TValue savetv2 = J2G(J)->tmptv2;
 656         setintV(L->top++, J->cur.traceno);
 657         setfuncV(L, L->top++, J->fn);
 658         setintV(L->top++, J->pt ? (int32_t)proto_bcpos(J->pt, J->pc) : -1);
 659         setintV(L->top++, J->framedepth);
 660       ,
 661         J2G(J)->tmptv = savetv;
 662         J2G(J)->tmptv2 = savetv2;
 663       );
 664       lj_record_ins(J);
 665       break;
 666 
 667     case LJ_TRACE_END:
 668       trace_pendpatch(J, 1);
 669       J->loopref = 0;
 670       if ((J->flags & JIT_F_OPT_LOOP) &&
 671           J->cur.link == J->cur.traceno && J->framedepth + J->retdepth == 0) {
 672         setvmstate(J2G(J), OPT);
 673         lj_opt_dce(J);
 674         if (lj_opt_loop(J)) {  /* Loop optimization failed? */
 675           J->cur.link = 0;
 676           J->cur.linktype = LJ_TRLINK_NONE;
 677           J->loopref = J->cur.nins;
 678           J->state = LJ_TRACE_RECORD;  /* Try to continue recording. */
 679           break;
 680         }
 681         J->loopref = J->chain[IR_LOOP];  /* Needed by assembler. */
 682       }
 683       lj_opt_split(J);
 684       lj_opt_sink(J);
 685       if (!J->loopref) J->cur.snap[J->cur.nsnap-1].count = SNAPCOUNT_DONE;
 686       J->state = LJ_TRACE_ASM;
 687       break;
 688 
 689     case LJ_TRACE_ASM:
 690       setvmstate(J2G(J), ASM);
 691       lj_asm_trace(J, &J->cur);
 692       trace_stop(J);
 693       setvmstate(J2G(J), INTERP);
 694       J->state = LJ_TRACE_IDLE;
 695       lj_dispatch_update(J2G(J));
 696       return NULL;
 697 
 698     default:  /* Trace aborted asynchronously. */
 699       setintV(L->top++, (int32_t)LJ_TRERR_RECERR);
 700       /* fallthrough */
 701     case LJ_TRACE_ERR:
 702       trace_pendpatch(J, 1);
 703       if (trace_abort(J))
 704         goto retry;
 705       setvmstate(J2G(J), INTERP);
 706       J->state = LJ_TRACE_IDLE;
 707       lj_dispatch_update(J2G(J));
 708       return NULL;
 709     }
 710   } while (J->state > LJ_TRACE_RECORD);
 711   return NULL;
 712 }
 713 
 714 /* -- Event handling ------------------------------------------------------ */
 715 
 716 /* A bytecode instruction is about to be executed. Record it. */
 717 void lj_trace_ins(jit_State *J, const BCIns *pc)
 718 {
 719   /* Note: J->L must already be set. pc is the true bytecode PC here. */
 720   J->pc = pc;
 721   J->fn = curr_func(J->L);
 722   J->pt = isluafunc(J->fn) ? funcproto(J->fn) : NULL;
 723   while (lj_vm_cpcall(J->L, NULL, (void *)J, trace_state) != 0)
 724     J->state = LJ_TRACE_ERR;
 725 }
 726 
 727 /* A hotcount triggered. Start recording a root trace. */
 728 void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc)
 729 {
 730   /* Note: pc is the interpreter bytecode PC here. It's offset by 1. */
 731   ERRNO_SAVE
 732   /* Reset hotcount. */
 733   hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]*HOTCOUNT_LOOP);
 734   /* Only start a new trace if not recording or inside __gc call or vmevent. */
 735   if (J->state == LJ_TRACE_IDLE &&
 736       !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
 737     J->parent = 0;  /* Root trace. */
 738     J->exitno = 0;
 739     J->state = LJ_TRACE_START;
 740     lj_trace_ins(J, pc-1);
 741   }
 742   ERRNO_RESTORE
 743 }
 744 
 745 /* Check for a hot side exit. If yes, start recording a side trace. */
 746 static void trace_hotside(jit_State *J, const BCIns *pc)
 747 {
 748   SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
 749   if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
 750       isluafunc(curr_func(J->L)) &&
 751       snap->count != SNAPCOUNT_DONE &&
 752       ++snap->count >= J->param[JIT_P_hotexit]) {
 753     lua_assert(J->state == LJ_TRACE_IDLE);
 754     /* J->parent is non-zero for a side trace. */
 755     J->state = LJ_TRACE_START;
 756     lj_trace_ins(J, pc);
 757   }
 758 }
 759 
 760 /* Stitch a new trace to the previous trace. */
 761 void LJ_FASTCALL lj_trace_stitch(jit_State *J, const BCIns *pc)
 762 {
 763   /* Only start a new trace if not recording or inside __gc call or vmevent. */
 764   if (J->state == LJ_TRACE_IDLE &&
 765       !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
 766     J->parent = 0;  /* Have to treat it like a root trace. */
 767     /* J->exitno is set to the invoking trace. */
 768     J->state = LJ_TRACE_START;
 769     lj_trace_ins(J, pc);
 770   }
 771 }
 772 
 773 
 774 /* Tiny struct to pass data to protected call. */
 775 typedef struct ExitDataCP {
 776   jit_State *J;
 777   void *exptr;          /* Pointer to exit state. */
 778   const BCIns *pc;      /* Restart interpreter at this PC. */
 779 } ExitDataCP;
 780 
 781 /* Need to protect lj_snap_restore because it may throw. */
 782 static TValue *trace_exit_cp(lua_State *L, lua_CFunction dummy, void *ud)
 783 {
 784   ExitDataCP *exd = (ExitDataCP *)ud;
 785   cframe_errfunc(L->cframe) = -1;  /* Inherit error function. */
 786   exd->pc = lj_snap_restore(exd->J, exd->exptr);
 787   UNUSED(dummy);
 788   return NULL;
 789 }
 790 
 791 #ifndef LUAJIT_DISABLE_VMEVENT
 792 /* Push all registers from exit state. */
 793 static void trace_exit_regs(lua_State *L, ExitState *ex)
 794 {
 795   int32_t i;
 796   setintV(L->top++, RID_NUM_GPR);
 797   setintV(L->top++, RID_NUM_FPR);
 798   for (i = 0; i < RID_NUM_GPR; i++) {
 799     if (sizeof(ex->gpr[i]) == sizeof(int32_t))
 800       setintV(L->top++, (int32_t)ex->gpr[i]);
 801     else
 802       setnumV(L->top++, (lua_Number)ex->gpr[i]);
 803   }
 804 #if !LJ_SOFTFP
 805   for (i = 0; i < RID_NUM_FPR; i++) {
 806     setnumV(L->top, ex->fpr[i]);
 807     if (LJ_UNLIKELY(tvisnan(L->top)))
 808       setnanV(L->top);
 809     L->top++;
 810   }
 811 #endif
 812 }
 813 #endif
 814 
 815 #ifdef EXITSTATE_PCREG
 816 /* Determine trace number from pc of exit instruction. */
 817 static TraceNo trace_exit_find(jit_State *J, MCode *pc)
 818 {
 819   TraceNo traceno;
 820   for (traceno = 1; traceno < J->sizetrace; traceno++) {
 821     GCtrace *T = traceref(J, traceno);
 822     if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode))
 823       return traceno;
 824   }
 825   lua_assert(0);
 826   return 0;
 827 }
 828 #endif
 829 
 830 /* A trace exited. Restore interpreter state. */
 831 int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
 832 {
 833   ERRNO_SAVE
 834   lua_State *L = J->L;
 835   ExitState *ex = (ExitState *)exptr;
 836   ExitDataCP exd;
 837   int errcode;
 838   const BCIns *pc;
 839   void *cf;
 840   GCtrace *T;
 841 #ifdef EXITSTATE_PCREG
 842   J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]);
 843 #endif
 844   T = traceref(J, J->parent); UNUSED(T);
 845 #ifdef EXITSTATE_CHECKEXIT
 846   if (J->exitno == T->nsnap) {  /* Treat stack check like a parent exit. */
 847     lua_assert(T->root != 0);
 848     J->exitno = T->ir[REF_BASE].op2;
 849     J->parent = T->ir[REF_BASE].op1;
 850     T = traceref(J, J->parent);
 851   }
 852 #endif
 853   lua_assert(T != NULL && J->exitno < T->nsnap);
 854   exd.J = J;
 855   exd.exptr = exptr;
 856   errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp);
 857   if (errcode)
 858     return -errcode;  /* Return negated error code. */
 859 
 860   if (!(LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)))
 861     lj_vmevent_send(L, TEXIT,
 862       lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK);
 863       setintV(L->top++, J->parent);
 864       setintV(L->top++, J->exitno);
 865       trace_exit_regs(L, ex);
 866     );
 867 
 868   pc = exd.pc;
 869   cf = cframe_raw(L->cframe);
 870   setcframe_pc(cf, pc);
 871   if (LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)) {
 872     /* Just exit to interpreter. */
 873   } else if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) {
 874     if (!(G(L)->hookmask & HOOK_GC))
 875       lj_gc_step(L);  /* Exited because of GC: drive GC forward. */
 876   } else {
 877     trace_hotside(J, pc);
 878   }
 879   if (bc_op(*pc) == BC_JLOOP) {
 880     BCIns *retpc = &traceref(J, bc_d(*pc))->startins;
 881     if (bc_isret(bc_op(*retpc))) {
 882       if (J->state == LJ_TRACE_RECORD) {
 883         J->patchins = *pc;
 884         J->patchpc = (BCIns *)pc;
 885         *J->patchpc = *retpc;
 886         J->bcskip = 1;
 887       } else {
 888         pc = retpc;
 889         setcframe_pc(cf, pc);
 890       }
 891     }
 892   }
 893   /* Return MULTRES or 0. */
 894   ERRNO_RESTORE
 895   switch (bc_op(*pc)) {
 896   case BC_CALLM: case BC_CALLMT:
 897     return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc) - LJ_FR2);
 898   case BC_RETM:
 899     return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc));
 900   case BC_TSETM:
 901     return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc));
 902   default:
 903     if (bc_op(*pc) >= BC_FUNCF)
 904       return (int)((BCReg)(L->top - L->base) + 1);
 905     return 0;
 906   }
 907 }
 908 
 909 #endif

/* [<][>][^][v][top][bottom][index][help] */