root/lj_crecord.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. argv2cdata
  2. crec_constructor
  3. argv2ctype
  4. crec_ct2irt
  5. crec_copy_struct
  6. crec_copy_unroll
  7. crec_copy_emit
  8. crec_copy
  9. crec_fill_unroll
  10. crec_fill_emit
  11. crec_fill
  12. crec_isnonzero
  13. crec_ct_ct
  14. crec_tv_ct
  15. crec_ct_tv
  16. crec_reassoc_ofs
  17. crec_tailcall
  18. crec_index_meta
  19. crec_index_bf
  20. recff_cdata_index
  21. crec_finalizer
  22. crec_alloc
  23. crec_call_args
  24. crec_snap_caller
  25. crec_call
  26. recff_cdata_call
  27. crec_arith_int64
  28. crec_arith_ptr
  29. crec_arith_meta
  30. recff_cdata_arith
  31. recff_clib_index
  32. crec_toint
  33. recff_ffi_new
  34. recff_ffi_errno
  35. recff_ffi_string
  36. recff_ffi_copy
  37. recff_ffi_fill
  38. recff_ffi_typeof
  39. recff_ffi_istype
  40. recff_ffi_abi
  41. recff_ffi_xof
  42. recff_ffi_gc
  43. crec_bit64_type
  44. recff_bit64_tobit
  45. recff_bit64_unary
  46. recff_bit64_nary
  47. recff_bit64_shift
  48. recff_bit64_tohex
  49. lj_crecord_tonumber

   1 /*
   2 ** Trace recorder for C data operations.
   3 ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5 
   6 #define lj_ffrecord_c
   7 #define LUA_CORE
   8 
   9 #include "lj_obj.h"
  10 
  11 #if LJ_HASJIT && LJ_HASFFI
  12 
  13 #include "lj_err.h"
  14 #include "lj_tab.h"
  15 #include "lj_frame.h"
  16 #include "lj_ctype.h"
  17 #include "lj_cdata.h"
  18 #include "lj_cparse.h"
  19 #include "lj_cconv.h"
  20 #include "lj_carith.h"
  21 #include "lj_clib.h"
  22 #include "lj_ccall.h"
  23 #include "lj_ff.h"
  24 #include "lj_ir.h"
  25 #include "lj_jit.h"
  26 #include "lj_ircall.h"
  27 #include "lj_iropt.h"
  28 #include "lj_trace.h"
  29 #include "lj_record.h"
  30 #include "lj_ffrecord.h"
  31 #include "lj_snap.h"
  32 #include "lj_crecord.h"
  33 #include "lj_dispatch.h"
  34 #include "lj_strfmt.h"
  35 
  36 /* Some local macros to save typing. Undef'd at the end. */
  37 #define IR(ref)                 (&J->cur.ir[(ref)])
  38 
  39 /* Pass IR on to next optimization in chain (FOLD). */
  40 #define emitir(ot, a, b)        (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
  41 
  42 #define emitconv(a, dt, st, flags) \
  43   emitir(IRT(IR_CONV, (dt)), (a), (st)|((dt) << 5)|(flags))
  44 
  45 /* -- C type checks ------------------------------------------------------- */
  46 
  47 static GCcdata *argv2cdata(jit_State *J, TRef tr, cTValue *o)
  48 {
  49   GCcdata *cd;
  50   TRef trtypeid;
  51   if (!tref_iscdata(tr))
  52     lj_trace_err(J, LJ_TRERR_BADTYPE);
  53   cd = cdataV(o);
  54   /* Specialize to the CTypeID. */
  55   trtypeid = emitir(IRT(IR_FLOAD, IRT_U16), tr, IRFL_CDATA_CTYPEID);
  56   emitir(IRTG(IR_EQ, IRT_INT), trtypeid, lj_ir_kint(J, (int32_t)cd->ctypeid));
  57   return cd;
  58 }
  59 
  60 /* Specialize to the CTypeID held by a cdata constructor. */
  61 static CTypeID crec_constructor(jit_State *J, GCcdata *cd, TRef tr)
  62 {
  63   CTypeID id;
  64   lua_assert(tref_iscdata(tr) && cd->ctypeid == CTID_CTYPEID);
  65   id = *(CTypeID *)cdataptr(cd);
  66   tr = emitir(IRT(IR_FLOAD, IRT_INT), tr, IRFL_CDATA_INT);
  67   emitir(IRTG(IR_EQ, IRT_INT), tr, lj_ir_kint(J, (int32_t)id));
  68   return id;
  69 }
  70 
  71 static CTypeID argv2ctype(jit_State *J, TRef tr, cTValue *o)
  72 {
  73   if (tref_isstr(tr)) {
  74     GCstr *s = strV(o);
  75     CPState cp;
  76     CTypeID oldtop;
  77     /* Specialize to the string containing the C type declaration. */
  78     emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, s));
  79     cp.L = J->L;
  80     cp.cts = ctype_ctsG(J2G(J));
  81     oldtop = cp.cts->top;
  82     cp.srcname = strdata(s);
  83     cp.p = strdata(s);
  84     cp.param = NULL;
  85     cp.mode = CPARSE_MODE_ABSTRACT|CPARSE_MODE_NOIMPLICIT;
  86     if (lj_cparse(&cp) || cp.cts->top > oldtop)  /* Avoid new struct defs. */
  87       lj_trace_err(J, LJ_TRERR_BADTYPE);
  88     return cp.val.id;
  89   } else {
  90     GCcdata *cd = argv2cdata(J, tr, o);
  91     return cd->ctypeid == CTID_CTYPEID ? crec_constructor(J, cd, tr) :
  92                                         cd->ctypeid;
  93   }
  94 }
  95 
  96 /* Convert CType to IRType (if possible). */
  97 static IRType crec_ct2irt(CTState *cts, CType *ct)
  98 {
  99   if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
 100   if (LJ_LIKELY(ctype_isnum(ct->info))) {
 101     if ((ct->info & CTF_FP)) {
 102       if (ct->size == sizeof(double))
 103         return IRT_NUM;
 104       else if (ct->size == sizeof(float))
 105         return IRT_FLOAT;
 106     } else {
 107       uint32_t b = lj_fls(ct->size);
 108       if (b <= 3)
 109         return IRT_I8 + 2*b + ((ct->info & CTF_UNSIGNED) ? 1 : 0);
 110     }
 111   } else if (ctype_isptr(ct->info)) {
 112     return (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
 113   } else if (ctype_iscomplex(ct->info)) {
 114     if (ct->size == 2*sizeof(double))
 115       return IRT_NUM;
 116     else if (ct->size == 2*sizeof(float))
 117       return IRT_FLOAT;
 118   }
 119   return IRT_CDATA;
 120 }
 121 
 122 /* -- Optimized memory fill and copy -------------------------------------- */
 123 
 124 /* Maximum length and unroll of inlined copy/fill. */
 125 #define CREC_COPY_MAXUNROLL             16
 126 #define CREC_COPY_MAXLEN                128
 127 
 128 #define CREC_FILL_MAXUNROLL             16
 129 
 130 /* Number of windowed registers used for optimized memory copy. */
 131 #if LJ_TARGET_X86
 132 #define CREC_COPY_REGWIN                2
 133 #elif LJ_TARGET_PPC || LJ_TARGET_MIPS
 134 #define CREC_COPY_REGWIN                8
 135 #else
 136 #define CREC_COPY_REGWIN                4
 137 #endif
 138 
 139 /* List of memory offsets for copy/fill. */
 140 typedef struct CRecMemList {
 141   CTSize ofs;           /* Offset in bytes. */
 142   IRType tp;            /* Type of load/store. */
 143   TRef trofs;           /* TRef of interned offset. */
 144   TRef trval;           /* TRef of load value. */
 145 } CRecMemList;
 146 
 147 /* Generate copy list for element-wise struct copy. */
 148 static MSize crec_copy_struct(CRecMemList *ml, CTState *cts, CType *ct)
 149 {
 150   CTypeID fid = ct->sib;
 151   MSize mlp = 0;
 152   while (fid) {
 153     CType *df = ctype_get(cts, fid);
 154     fid = df->sib;
 155     if (ctype_isfield(df->info)) {
 156       CType *cct;
 157       IRType tp;
 158       if (!gcref(df->name)) continue;  /* Ignore unnamed fields. */
 159       cct = ctype_rawchild(cts, df);  /* Field type. */
 160       tp = crec_ct2irt(cts, cct);
 161       if (tp == IRT_CDATA) return 0;  /* NYI: aggregates. */
 162       if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 163       ml[mlp].ofs = df->size;
 164       ml[mlp].tp = tp;
 165       mlp++;
 166       if (ctype_iscomplex(cct->info)) {
 167         if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 168         ml[mlp].ofs = df->size + (cct->size >> 1);
 169         ml[mlp].tp = tp;
 170         mlp++;
 171       }
 172     } else if (!ctype_isconstval(df->info)) {
 173       /* NYI: bitfields and sub-structures. */
 174       return 0;
 175     }
 176   }
 177   return mlp;
 178 }
 179 
 180 /* Generate unrolled copy list, from highest to lowest step size/alignment. */
 181 static MSize crec_copy_unroll(CRecMemList *ml, CTSize len, CTSize step,
 182                               IRType tp)
 183 {
 184   CTSize ofs = 0;
 185   MSize mlp = 0;
 186   if (tp == IRT_CDATA) tp = IRT_U8 + 2*lj_fls(step);
 187   do {
 188     while (ofs + step <= len) {
 189       if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 190       ml[mlp].ofs = ofs;
 191       ml[mlp].tp = tp;
 192       mlp++;
 193       ofs += step;
 194     }
 195     step >>= 1;
 196     tp -= 2;
 197   } while (ofs < len);
 198   return mlp;
 199 }
 200 
 201 /*
 202 ** Emit copy list with windowed loads/stores.
 203 ** LJ_TARGET_UNALIGNED: may emit unaligned loads/stores (not marked as such).
 204 */
 205 static void crec_copy_emit(jit_State *J, CRecMemList *ml, MSize mlp,
 206                            TRef trdst, TRef trsrc)
 207 {
 208   MSize i, j, rwin = 0;
 209   for (i = 0, j = 0; i < mlp; ) {
 210     TRef trofs = lj_ir_kintp(J, ml[i].ofs);
 211     TRef trsptr = emitir(IRT(IR_ADD, IRT_PTR), trsrc, trofs);
 212     ml[i].trval = emitir(IRT(IR_XLOAD, ml[i].tp), trsptr, 0);
 213     ml[i].trofs = trofs;
 214     i++;
 215     rwin += (LJ_SOFTFP32 && ml[i].tp == IRT_NUM) ? 2 : 1;
 216     if (rwin >= CREC_COPY_REGWIN || i >= mlp) {  /* Flush buffered stores. */
 217       rwin = 0;
 218       for ( ; j < i; j++) {
 219         TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, ml[j].trofs);
 220         emitir(IRT(IR_XSTORE, ml[j].tp), trdptr, ml[j].trval);
 221       }
 222     }
 223   }
 224 }
 225 
 226 /* Optimized memory copy. */
 227 static void crec_copy(jit_State *J, TRef trdst, TRef trsrc, TRef trlen,
 228                       CType *ct)
 229 {
 230   if (tref_isk(trlen)) {  /* Length must be constant. */
 231     CRecMemList ml[CREC_COPY_MAXUNROLL];
 232     MSize mlp = 0;
 233     CTSize step = 1, len = (CTSize)IR(tref_ref(trlen))->i;
 234     IRType tp = IRT_CDATA;
 235     int needxbar = 0;
 236     if (len == 0) return;  /* Shortcut. */
 237     if (len > CREC_COPY_MAXLEN) goto fallback;
 238     if (ct) {
 239       CTState *cts = ctype_ctsG(J2G(J));
 240       lua_assert(ctype_isarray(ct->info) || ctype_isstruct(ct->info));
 241       if (ctype_isarray(ct->info)) {
 242         CType *cct = ctype_rawchild(cts, ct);
 243         tp = crec_ct2irt(cts, cct);
 244         if (tp == IRT_CDATA) goto rawcopy;
 245         step = lj_ir_type_size[tp];
 246         lua_assert((len & (step-1)) == 0);
 247       } else if ((ct->info & CTF_UNION)) {
 248         step = (1u << ctype_align(ct->info));
 249         goto rawcopy;
 250       } else {
 251         mlp = crec_copy_struct(ml, cts, ct);
 252         goto emitcopy;
 253       }
 254     } else {
 255     rawcopy:
 256       needxbar = 1;
 257       if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
 258         step = CTSIZE_PTR;
 259     }
 260     mlp = crec_copy_unroll(ml, len, step, tp);
 261   emitcopy:
 262     if (mlp) {
 263       crec_copy_emit(J, ml, mlp, trdst, trsrc);
 264       if (needxbar)
 265         emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
 266       return;
 267     }
 268   }
 269 fallback:
 270   /* Call memcpy. Always needs a barrier to disable alias analysis. */
 271   lj_ir_call(J, IRCALL_memcpy, trdst, trsrc, trlen);
 272   emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
 273 }
 274 
 275 /* Generate unrolled fill list, from highest to lowest step size/alignment. */
 276 static MSize crec_fill_unroll(CRecMemList *ml, CTSize len, CTSize step)
 277 {
 278   CTSize ofs = 0;
 279   MSize mlp = 0;
 280   IRType tp = IRT_U8 + 2*lj_fls(step);
 281   do {
 282     while (ofs + step <= len) {
 283       if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 284       ml[mlp].ofs = ofs;
 285       ml[mlp].tp = tp;
 286       mlp++;
 287       ofs += step;
 288     }
 289     step >>= 1;
 290     tp -= 2;
 291   } while (ofs < len);
 292   return mlp;
 293 }
 294 
 295 /*
 296 ** Emit stores for fill list.
 297 ** LJ_TARGET_UNALIGNED: may emit unaligned stores (not marked as such).
 298 */
 299 static void crec_fill_emit(jit_State *J, CRecMemList *ml, MSize mlp,
 300                            TRef trdst, TRef trfill)
 301 {
 302   MSize i;
 303   for (i = 0; i < mlp; i++) {
 304     TRef trofs = lj_ir_kintp(J, ml[i].ofs);
 305     TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, trofs);
 306     emitir(IRT(IR_XSTORE, ml[i].tp), trdptr, trfill);
 307   }
 308 }
 309 
 310 /* Optimized memory fill. */
 311 static void crec_fill(jit_State *J, TRef trdst, TRef trlen, TRef trfill,
 312                       CTSize step)
 313 {
 314   if (tref_isk(trlen)) {  /* Length must be constant. */
 315     CRecMemList ml[CREC_FILL_MAXUNROLL];
 316     MSize mlp;
 317     CTSize len = (CTSize)IR(tref_ref(trlen))->i;
 318     if (len == 0) return;  /* Shortcut. */
 319     if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
 320       step = CTSIZE_PTR;
 321     if (step * CREC_FILL_MAXUNROLL < len) goto fallback;
 322     mlp = crec_fill_unroll(ml, len, step);
 323     if (!mlp) goto fallback;
 324     if (tref_isk(trfill) || ml[0].tp != IRT_U8)
 325       trfill = emitconv(trfill, IRT_INT, IRT_U8, 0);
 326     if (ml[0].tp != IRT_U8) {  /* Scatter U8 to U16/U32/U64. */
 327       if (CTSIZE_PTR == 8 && ml[0].tp == IRT_U64) {
 328         if (tref_isk(trfill))  /* Pointless on x64 with zero-extended regs. */
 329           trfill = emitconv(trfill, IRT_U64, IRT_U32, 0);
 330         trfill = emitir(IRT(IR_MUL, IRT_U64), trfill,
 331                         lj_ir_kint64(J, U64x(01010101,01010101)));
 332       } else {
 333         trfill = emitir(IRTI(IR_MUL), trfill,
 334                    lj_ir_kint(J, ml[0].tp == IRT_U16 ? 0x0101 : 0x01010101));
 335       }
 336     }
 337     crec_fill_emit(J, ml, mlp, trdst, trfill);
 338   } else {
 339 fallback:
 340     /* Call memset. Always needs a barrier to disable alias analysis. */
 341     lj_ir_call(J, IRCALL_memset, trdst, trfill, trlen);  /* Note: arg order! */
 342   }
 343   emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
 344 }
 345 
 346 /* -- Convert C type to C type -------------------------------------------- */
 347 
 348 /*
 349 ** This code mirrors the code in lj_cconv.c. It performs the same steps
 350 ** for the trace recorder that lj_cconv.c does for the interpreter.
 351 **
 352 ** One major difference is that we can get away with much fewer checks
 353 ** here. E.g. checks for casts, constness or correct types can often be
 354 ** omitted, even if they might fail. The interpreter subsequently throws
 355 ** an error, which aborts the trace.
 356 **
 357 ** All operations are specialized to their C types, so the on-trace
 358 ** outcome must be the same as the outcome in the interpreter. If the
 359 ** interpreter doesn't throw an error, then the trace is correct, too.
 360 ** Care must be taken not to generate invalid (temporary) IR or to
 361 ** trigger asserts.
 362 */
 363 
 364 /* Determine whether a passed number or cdata number is non-zero. */
 365 static int crec_isnonzero(CType *s, void *p)
 366 {
 367   if (p == (void *)0)
 368     return 0;
 369   if (p == (void *)1)
 370     return 1;
 371   if ((s->info & CTF_FP)) {
 372     if (s->size == sizeof(float))
 373       return (*(float *)p != 0);
 374     else
 375       return (*(double *)p != 0);
 376   } else {
 377     if (s->size == 1)
 378       return (*(uint8_t *)p != 0);
 379     else if (s->size == 2)
 380       return (*(uint16_t *)p != 0);
 381     else if (s->size == 4)
 382       return (*(uint32_t *)p != 0);
 383     else
 384       return (*(uint64_t *)p != 0);
 385   }
 386 }
 387 
 388 static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
 389                        void *svisnz)
 390 {
 391   IRType dt = crec_ct2irt(ctype_ctsG(J2G(J)), d);
 392   IRType st = crec_ct2irt(ctype_ctsG(J2G(J)), s);
 393   CTSize dsize = d->size, ssize = s->size;
 394   CTInfo dinfo = d->info, sinfo = s->info;
 395 
 396   if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT)
 397     goto err_conv;
 398 
 399   /*
 400   ** Note: Unlike lj_cconv_ct_ct(), sp holds the _value_ of pointers and
 401   ** numbers up to 8 bytes. Otherwise sp holds a pointer.
 402   */
 403 
 404   switch (cconv_idx2(dinfo, sinfo)) {
 405   /* Destination is a bool. */
 406   case CCX(B, B):
 407     goto xstore;  /* Source operand is already normalized. */
 408   case CCX(B, I):
 409   case CCX(B, F):
 410     if (st != IRT_CDATA) {
 411       /* Specialize to the result of a comparison against 0. */
 412       TRef zero = (st == IRT_NUM  || st == IRT_FLOAT) ? lj_ir_knum(J, 0) :
 413                   (st == IRT_I64 || st == IRT_U64) ? lj_ir_kint64(J, 0) :
 414                   lj_ir_kint(J, 0);
 415       int isnz = crec_isnonzero(s, svisnz);
 416       emitir(IRTG(isnz ? IR_NE : IR_EQ, st), sp, zero);
 417       sp = lj_ir_kint(J, isnz);
 418       goto xstore;
 419     }
 420     goto err_nyi;
 421 
 422   /* Destination is an integer. */
 423   case CCX(I, B):
 424   case CCX(I, I):
 425   conv_I_I:
 426     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 427     /* Extend 32 to 64 bit integer. */
 428     if (dsize == 8 && ssize < 8 && !(LJ_64 && (sinfo & CTF_UNSIGNED)))
 429       sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st,
 430                     (sinfo & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
 431     else if (dsize < 8 && ssize == 8)  /* Truncate from 64 bit integer. */
 432       sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, 0);
 433     else if (st == IRT_INT)
 434       sp = lj_opt_narrow_toint(J, sp);
 435   xstore:
 436     if (dt == IRT_I64 || dt == IRT_U64) lj_needsplit(J);
 437     if (dp == 0) return sp;
 438     emitir(IRT(IR_XSTORE, dt), dp, sp);
 439     break;
 440   case CCX(I, C):
 441     sp = emitir(IRT(IR_XLOAD, st), sp, 0);  /* Load re. */
 442     /* fallthrough */
 443   case CCX(I, F):
 444     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 445     sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_ANY);
 446     goto xstore;
 447   case CCX(I, P):
 448   case CCX(I, A):
 449     sinfo = CTINFO(CT_NUM, CTF_UNSIGNED);
 450     ssize = CTSIZE_PTR;
 451     st = IRT_UINTP;
 452     if (((dsize ^ ssize) & 8) == 0) {  /* Must insert no-op type conversion. */
 453       sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, IRT_PTR, 0);
 454       goto xstore;
 455     }
 456     goto conv_I_I;
 457 
 458   /* Destination is a floating-point number. */
 459   case CCX(F, B):
 460   case CCX(F, I):
 461   conv_F_I:
 462     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 463     sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st, 0);
 464     goto xstore;
 465   case CCX(F, C):
 466     sp = emitir(IRT(IR_XLOAD, st), sp, 0);  /* Load re. */
 467     /* fallthrough */
 468   case CCX(F, F):
 469   conv_F_F:
 470     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 471     if (dt != st) sp = emitconv(sp, dt, st, 0);
 472     goto xstore;
 473 
 474   /* Destination is a complex number. */
 475   case CCX(C, I):
 476   case CCX(C, F):
 477     {  /* Clear im. */
 478       TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
 479       emitir(IRT(IR_XSTORE, dt), ptr, lj_ir_knum(J, 0));
 480     }
 481     /* Convert to re. */
 482     if ((sinfo & CTF_FP)) goto conv_F_F; else goto conv_F_I;
 483 
 484   case CCX(C, C):
 485     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 486     {
 487       TRef re, im, ptr;
 488       re = emitir(IRT(IR_XLOAD, st), sp, 0);
 489       ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, (ssize >> 1)));
 490       im = emitir(IRT(IR_XLOAD, st), ptr, 0);
 491       if (dt != st) {
 492         re = emitconv(re, dt, st, 0);
 493         im = emitconv(im, dt, st, 0);
 494       }
 495       emitir(IRT(IR_XSTORE, dt), dp, re);
 496       ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
 497       emitir(IRT(IR_XSTORE, dt), ptr, im);
 498     }
 499     break;
 500 
 501   /* Destination is a vector. */
 502   case CCX(V, I):
 503   case CCX(V, F):
 504   case CCX(V, C):
 505   case CCX(V, V):
 506     goto err_nyi;
 507 
 508   /* Destination is a pointer. */
 509   case CCX(P, P):
 510   case CCX(P, A):
 511   case CCX(P, S):
 512     /* There are only 32 bit pointers/addresses on 32 bit machines.
 513     ** Also ok on x64, since all 32 bit ops clear the upper part of the reg.
 514     */
 515     goto xstore;
 516   case CCX(P, I):
 517     if (st == IRT_CDATA) goto err_nyi;
 518     if (!LJ_64 && ssize == 8)  /* Truncate from 64 bit integer. */
 519       sp = emitconv(sp, IRT_U32, st, 0);
 520     goto xstore;
 521   case CCX(P, F):
 522     if (st == IRT_CDATA) goto err_nyi;
 523     /* The signed conversion is cheaper. x64 really has 47 bit pointers. */
 524     sp = emitconv(sp, (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32,
 525                   st, IRCONV_ANY);
 526     goto xstore;
 527 
 528   /* Destination is an array. */
 529   case CCX(A, A):
 530   /* Destination is a struct/union. */
 531   case CCX(S, S):
 532     if (dp == 0) goto err_conv;
 533     crec_copy(J, dp, sp, lj_ir_kint(J, dsize), d);
 534     break;
 535 
 536   default:
 537   err_conv:
 538   err_nyi:
 539     lj_trace_err(J, LJ_TRERR_NYICONV);
 540     break;
 541   }
 542   return 0;
 543 }
 544 
 545 /* -- Convert C type to TValue (load) ------------------------------------- */
 546 
 547 static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
 548 {
 549   CTState *cts = ctype_ctsG(J2G(J));
 550   IRType t = crec_ct2irt(cts, s);
 551   CTInfo sinfo = s->info;
 552   if (ctype_isnum(sinfo)) {
 553     TRef tr;
 554     if (t == IRT_CDATA)
 555       goto err_nyi;  /* NYI: copyval of >64 bit integers. */
 556     tr = emitir(IRT(IR_XLOAD, t), sp, 0);
 557     if (t == IRT_FLOAT || t == IRT_U32) {  /* Keep uint32_t/float as numbers. */
 558       return emitconv(tr, IRT_NUM, t, 0);
 559     } else if (t == IRT_I64 || t == IRT_U64) {  /* Box 64 bit integer. */
 560       sp = tr;
 561       lj_needsplit(J);
 562     } else if ((sinfo & CTF_BOOL)) {
 563       /* Assume not equal to zero. Fixup and emit pending guard later. */
 564       lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
 565       J->postproc = LJ_POST_FIXGUARD;
 566       return TREF_TRUE;
 567     } else {
 568       return tr;
 569     }
 570   } else if (ctype_isptr(sinfo) || ctype_isenum(sinfo)) {
 571     sp = emitir(IRT(IR_XLOAD, t), sp, 0);  /* Box pointers and enums. */
 572   } else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) {
 573     cts->L = J->L;
 574     sid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR);  /* Create ref. */
 575   } else if (ctype_iscomplex(sinfo)) {  /* Unbox/box complex. */
 576     ptrdiff_t esz = (ptrdiff_t)(s->size >> 1);
 577     TRef ptr, tr1, tr2, dp;
 578     dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL);
 579     tr1 = emitir(IRT(IR_XLOAD, t), sp, 0);
 580     ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, esz));
 581     tr2 = emitir(IRT(IR_XLOAD, t), ptr, 0);
 582     ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)));
 583     emitir(IRT(IR_XSTORE, t), ptr, tr1);
 584     ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)+esz));
 585     emitir(IRT(IR_XSTORE, t), ptr, tr2);
 586     return dp;
 587   } else {
 588     /* NYI: copyval of vectors. */
 589   err_nyi:
 590     lj_trace_err(J, LJ_TRERR_NYICONV);
 591   }
 592   /* Box pointer, ref, enum or 64 bit integer. */
 593   return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, sid), sp);
 594 }
 595 
 596 /* -- Convert TValue to C type (store) ------------------------------------ */
 597 
 598 static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
 599 {
 600   CTState *cts = ctype_ctsG(J2G(J));
 601   CTypeID sid = CTID_P_VOID;
 602   void *svisnz = 0;
 603   CType *s;
 604   if (LJ_LIKELY(tref_isinteger(sp))) {
 605     sid = CTID_INT32;
 606     svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
 607   } else if (tref_isnum(sp)) {
 608     sid = CTID_DOUBLE;
 609     svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
 610   } else if (tref_isbool(sp)) {
 611     sp = lj_ir_kint(J, tref_istrue(sp) ? 1 : 0);
 612     sid = CTID_BOOL;
 613   } else if (tref_isnil(sp)) {
 614     sp = lj_ir_kptr(J, NULL);
 615   } else if (tref_isudata(sp)) {
 616     GCudata *ud = udataV(sval);
 617     if (ud->udtype == UDTYPE_IO_FILE) {
 618       TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), sp, IRFL_UDATA_UDTYPE);
 619       emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, UDTYPE_IO_FILE));
 620       sp = emitir(IRT(IR_FLOAD, IRT_PTR), sp, IRFL_UDATA_FILE);
 621     } else {
 622       sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCudata)));
 623     }
 624   } else if (tref_isstr(sp)) {
 625     if (ctype_isenum(d->info)) {  /* Match string against enum constant. */
 626       GCstr *str = strV(sval);
 627       CTSize ofs;
 628       CType *cct = lj_ctype_getfield(cts, d, str, &ofs);
 629       /* Specialize to the name of the enum constant. */
 630       emitir(IRTG(IR_EQ, IRT_STR), sp, lj_ir_kstr(J, str));
 631       if (cct && ctype_isconstval(cct->info)) {
 632         lua_assert(ctype_child(cts, cct)->size == 4);
 633         svisnz = (void *)(intptr_t)(ofs != 0);
 634         sp = lj_ir_kint(J, (int32_t)ofs);
 635         sid = ctype_cid(cct->info);
 636       }  /* else: interpreter will throw. */
 637     } else if (ctype_isrefarray(d->info)) {  /* Copy string to array. */
 638       lj_trace_err(J, LJ_TRERR_BADTYPE);  /* NYI */
 639     } else {  /* Otherwise pass the string data as a const char[]. */
 640       /* Don't use STRREF. It folds with SNEW, which loses the trailing NUL. */
 641       sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCstr)));
 642       sid = CTID_A_CCHAR;
 643     }
 644   } else if (tref_islightud(sp)) {
 645 #if LJ_64
 646     sp = emitir(IRT(IR_BAND, IRT_P64), sp,
 647                 lj_ir_kint64(J, U64x(00007fff,ffffffff)));
 648 #endif
 649   } else {  /* NYI: tref_istab(sp). */
 650     IRType t;
 651     sid = argv2cdata(J, sp, sval)->ctypeid;
 652     s = ctype_raw(cts, sid);
 653     svisnz = cdataptr(cdataV(sval));
 654     if (ctype_isfunc(s->info)) {
 655       sid = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|sid), CTSIZE_PTR);
 656       s = ctype_get(cts, sid);
 657       t = IRT_PTR;
 658     } else {
 659       t = crec_ct2irt(cts, s);
 660     }
 661     if (ctype_isptr(s->info)) {
 662       sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_PTR);
 663       if (ctype_isref(s->info)) {
 664         svisnz = *(void **)svisnz;
 665         s = ctype_rawchild(cts, s);
 666         if (ctype_isenum(s->info)) s = ctype_child(cts, s);
 667         t = crec_ct2irt(cts, s);
 668       } else {
 669         goto doconv;
 670       }
 671     } else if (t == IRT_I64 || t == IRT_U64) {
 672       sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT64);
 673       lj_needsplit(J);
 674       goto doconv;
 675     } else if (t == IRT_INT || t == IRT_U32) {
 676       if (ctype_isenum(s->info)) s = ctype_child(cts, s);
 677       sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT);
 678       goto doconv;
 679     } else {
 680       sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCcdata)));
 681     }
 682     if (ctype_isnum(s->info) && t != IRT_CDATA)
 683       sp = emitir(IRT(IR_XLOAD, t), sp, 0);  /* Load number value. */
 684     goto doconv;
 685   }
 686   s = ctype_get(cts, sid);
 687 doconv:
 688   if (ctype_isenum(d->info)) d = ctype_child(cts, d);
 689   return crec_ct_ct(J, d, s, dp, sp, svisnz);
 690 }
 691 
 692 /* -- C data metamethods -------------------------------------------------- */
 693 
 694 /* This would be rather difficult in FOLD, so do it here:
 695 ** (base+k)+(idx*sz)+ofs ==> (base+idx*sz)+(ofs+k)
 696 ** (base+(idx+k)*sz)+ofs ==> (base+idx*sz)+(ofs+k*sz)
 697 */
 698 static TRef crec_reassoc_ofs(jit_State *J, TRef tr, ptrdiff_t *ofsp, MSize sz)
 699 {
 700   IRIns *ir = IR(tref_ref(tr));
 701   if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD) && irref_isk(ir->op2) &&
 702       (ir->o == IR_ADD || ir->o == IR_ADDOV || ir->o == IR_SUBOV)) {
 703     IRIns *irk = IR(ir->op2);
 704     ptrdiff_t k;
 705     if (LJ_64 && irk->o == IR_KINT64)
 706       k = (ptrdiff_t)ir_kint64(irk)->u64 * sz;
 707     else
 708       k = (ptrdiff_t)irk->i * sz;
 709     if (ir->o == IR_SUBOV) *ofsp -= k; else *ofsp += k;
 710     tr = ir->op1;  /* Not a TRef, but the caller doesn't care. */
 711   }
 712   return tr;
 713 }
 714 
 715 /* Tailcall to function. */
 716 static void crec_tailcall(jit_State *J, RecordFFData *rd, cTValue *tv)
 717 {
 718   TRef kfunc = lj_ir_kfunc(J, funcV(tv));
 719 #if LJ_FR2
 720   J->base[-2] = kfunc;
 721   J->base[-1] = TREF_FRAME;
 722 #else
 723   J->base[-1] = kfunc | TREF_FRAME;
 724 #endif
 725   rd->nres = -1;  /* Pending tailcall. */
 726 }
 727 
 728 /* Record ctype __index/__newindex metamethods. */
 729 static void crec_index_meta(jit_State *J, CTState *cts, CType *ct,
 730                             RecordFFData *rd)
 731 {
 732   CTypeID id = ctype_typeid(cts, ct);
 733   cTValue *tv = lj_ctype_meta(cts, id, rd->data ? MM_newindex : MM_index);
 734   if (!tv)
 735     lj_trace_err(J, LJ_TRERR_BADTYPE);
 736   if (tvisfunc(tv)) {
 737     crec_tailcall(J, rd, tv);
 738   } else if (rd->data == 0 && tvistab(tv) && tref_isstr(J->base[1])) {
 739     /* Specialize to result of __index lookup. */
 740     cTValue *o = lj_tab_get(J->L, tabV(tv), &rd->argv[1]);
 741     J->base[0] = lj_record_constify(J, o);
 742     if (!J->base[0])
 743       lj_trace_err(J, LJ_TRERR_BADTYPE);
 744     /* Always specialize to the key. */
 745     emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
 746   } else {
 747     /* NYI: resolving of non-function metamethods. */
 748     /* NYI: non-string keys for __index table. */
 749     /* NYI: stores to __newindex table. */
 750     lj_trace_err(J, LJ_TRERR_BADTYPE);
 751   }
 752 }
 753 
 754 /* Record bitfield load/store. */
 755 static void crec_index_bf(jit_State *J, RecordFFData *rd, TRef ptr, CTInfo info)
 756 {
 757   IRType t = IRT_I8 + 2*lj_fls(ctype_bitcsz(info)) + ((info&CTF_UNSIGNED)?1:0);
 758   TRef tr = emitir(IRT(IR_XLOAD, t), ptr, 0);
 759   CTSize pos = ctype_bitpos(info), bsz = ctype_bitbsz(info), shift = 32 - bsz;
 760   lua_assert(t <= IRT_U32);  /* NYI: 64 bit bitfields. */
 761   if (rd->data == 0) {  /* __index metamethod. */
 762     if ((info & CTF_BOOL)) {
 763       tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << pos))));
 764       /* Assume not equal to zero. Fixup and emit pending guard later. */
 765       lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
 766       J->postproc = LJ_POST_FIXGUARD;
 767       tr = TREF_TRUE;
 768     } else if (!(info & CTF_UNSIGNED)) {
 769       tr = emitir(IRTI(IR_BSHL), tr, lj_ir_kint(J, shift - pos));
 770       tr = emitir(IRTI(IR_BSAR), tr, lj_ir_kint(J, shift));
 771     } else {
 772       lua_assert(bsz < 32);  /* Full-size fields cannot end up here. */
 773       tr = emitir(IRTI(IR_BSHR), tr, lj_ir_kint(J, pos));
 774       tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << bsz)-1)));
 775       /* We can omit the U32 to NUM conversion, since bsz < 32. */
 776     }
 777     J->base[0] = tr;
 778   } else {  /* __newindex metamethod. */
 779     CTState *cts = ctype_ctsG(J2G(J));
 780     CType *ct = ctype_get(cts,
 781                           (info & CTF_BOOL) ? CTID_BOOL :
 782                           (info & CTF_UNSIGNED) ? CTID_UINT32 : CTID_INT32);
 783     int32_t mask = (int32_t)(((1u << bsz)-1) << pos);
 784     TRef sp = crec_ct_tv(J, ct, 0, J->base[2], &rd->argv[2]);
 785     sp = emitir(IRTI(IR_BSHL), sp, lj_ir_kint(J, pos));
 786     /* Use of the target type avoids forwarding conversions. */
 787     sp = emitir(IRT(IR_BAND, t), sp, lj_ir_kint(J, mask));
 788     tr = emitir(IRT(IR_BAND, t), tr, lj_ir_kint(J, (int32_t)~mask));
 789     tr = emitir(IRT(IR_BOR, t), tr, sp);
 790     emitir(IRT(IR_XSTORE, t), ptr, tr);
 791     rd->nres = 0;
 792     J->needsnap = 1;
 793   }
 794 }
 795 
 796 void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd)
 797 {
 798   TRef idx, ptr = J->base[0];
 799   ptrdiff_t ofs = sizeof(GCcdata);
 800   GCcdata *cd = argv2cdata(J, ptr, &rd->argv[0]);
 801   CTState *cts = ctype_ctsG(J2G(J));
 802   CType *ct = ctype_raw(cts, cd->ctypeid);
 803   CTypeID sid = 0;
 804 
 805   /* Resolve pointer or reference for cdata object. */
 806   if (ctype_isptr(ct->info)) {
 807     IRType t = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
 808     if (ctype_isref(ct->info)) ct = ctype_rawchild(cts, ct);
 809     ptr = emitir(IRT(IR_FLOAD, t), ptr, IRFL_CDATA_PTR);
 810     ofs = 0;
 811     ptr = crec_reassoc_ofs(J, ptr, &ofs, 1);
 812   }
 813 
 814 again:
 815   idx = J->base[1];
 816   if (tref_isnumber(idx)) {
 817     idx = lj_opt_narrow_cindex(J, idx);
 818     if (ctype_ispointer(ct->info)) {
 819       CTSize sz;
 820   integer_key:
 821       if ((ct->info & CTF_COMPLEX))
 822         idx = emitir(IRT(IR_BAND, IRT_INTP), idx, lj_ir_kintp(J, 1));
 823       sz = lj_ctype_size(cts, (sid = ctype_cid(ct->info)));
 824       idx = crec_reassoc_ofs(J, idx, &ofs, sz);
 825 #if LJ_TARGET_ARM || LJ_TARGET_PPC
 826       /* Hoist base add to allow fusion of index/shift into operands. */
 827       if (LJ_LIKELY(J->flags & JIT_F_OPT_LOOP) && ofs
 828 #if LJ_TARGET_ARM
 829           && (sz == 1 || sz == 4)
 830 #endif
 831           ) {
 832         ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
 833         ofs = 0;
 834       }
 835 #endif
 836       idx = emitir(IRT(IR_MUL, IRT_INTP), idx, lj_ir_kintp(J, sz));
 837       ptr = emitir(IRT(IR_ADD, IRT_PTR), idx, ptr);
 838     }
 839   } else if (tref_iscdata(idx)) {
 840     GCcdata *cdk = cdataV(&rd->argv[1]);
 841     CType *ctk = ctype_raw(cts, cdk->ctypeid);
 842     IRType t = crec_ct2irt(cts, ctk);
 843     if (ctype_ispointer(ct->info) && t >= IRT_I8 && t <= IRT_U64) {
 844       if (ctk->size == 8) {
 845         idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT64);
 846       } else if (ctk->size == 4) {
 847         idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT);
 848       } else {
 849         idx = emitir(IRT(IR_ADD, IRT_PTR), idx,
 850                      lj_ir_kintp(J, sizeof(GCcdata)));
 851         idx = emitir(IRT(IR_XLOAD, t), idx, 0);
 852       }
 853       if (LJ_64 && ctk->size < sizeof(intptr_t) && !(ctk->info & CTF_UNSIGNED))
 854         idx = emitconv(idx, IRT_INTP, IRT_INT, IRCONV_SEXT);
 855       if (!LJ_64 && ctk->size > sizeof(intptr_t)) {
 856         idx = emitconv(idx, IRT_INTP, t, 0);
 857         lj_needsplit(J);
 858       }
 859       goto integer_key;
 860     }
 861   } else if (tref_isstr(idx)) {
 862     GCstr *name = strV(&rd->argv[1]);
 863     if (cd && cd->ctypeid == CTID_CTYPEID)
 864       ct = ctype_raw(cts, crec_constructor(J, cd, ptr));
 865     if (ctype_isstruct(ct->info)) {
 866       CTSize fofs;
 867       CType *fct;
 868       fct = lj_ctype_getfield(cts, ct, name, &fofs);
 869       if (fct) {
 870         ofs += (ptrdiff_t)fofs;
 871         /* Always specialize to the field name. */
 872         emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
 873         if (ctype_isconstval(fct->info)) {
 874           if (fct->size >= 0x80000000u &&
 875               (ctype_child(cts, fct)->info & CTF_UNSIGNED)) {
 876             J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)fct->size);
 877             return;
 878           }
 879           J->base[0] = lj_ir_kint(J, (int32_t)fct->size);
 880           return;  /* Interpreter will throw for newindex. */
 881         } else if (ctype_isbitfield(fct->info)) {
 882           if (ofs)
 883             ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
 884           crec_index_bf(J, rd, ptr, fct->info);
 885           return;
 886         } else {
 887           lua_assert(ctype_isfield(fct->info));
 888           sid = ctype_cid(fct->info);
 889         }
 890       }
 891     } else if (ctype_iscomplex(ct->info)) {
 892       if (name->len == 2 &&
 893           ((strdata(name)[0] == 'r' && strdata(name)[1] == 'e') ||
 894            (strdata(name)[0] == 'i' && strdata(name)[1] == 'm'))) {
 895         /* Always specialize to the field name. */
 896         emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
 897         if (strdata(name)[0] == 'i') ofs += (ct->size >> 1);
 898         sid = ctype_cid(ct->info);
 899       }
 900     }
 901   }
 902   if (!sid) {
 903     if (ctype_isptr(ct->info)) {  /* Automatically perform '->'. */
 904       CType *cct = ctype_rawchild(cts, ct);
 905       if (ctype_isstruct(cct->info)) {
 906         ct = cct;
 907         cd = NULL;
 908         if (tref_isstr(idx)) goto again;
 909       }
 910     }
 911     crec_index_meta(J, cts, ct, rd);
 912     return;
 913   }
 914 
 915   if (ofs)
 916     ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
 917 
 918   /* Resolve reference for field. */
 919   ct = ctype_get(cts, sid);
 920   if (ctype_isref(ct->info)) {
 921     ptr = emitir(IRT(IR_XLOAD, IRT_PTR), ptr, 0);
 922     sid = ctype_cid(ct->info);
 923     ct = ctype_get(cts, sid);
 924   }
 925 
 926   while (ctype_isattrib(ct->info))
 927     ct = ctype_child(cts, ct);  /* Skip attributes. */
 928 
 929   if (rd->data == 0) {  /* __index metamethod. */
 930     J->base[0] = crec_tv_ct(J, ct, sid, ptr);
 931   } else {  /* __newindex metamethod. */
 932     rd->nres = 0;
 933     J->needsnap = 1;
 934     crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
 935   }
 936 }
 937 
 938 /* Record setting a finalizer. */
 939 static void crec_finalizer(jit_State *J, TRef trcd, TRef trfin, cTValue *fin)
 940 {
 941   if (tvisgcv(fin)) {
 942     if (!trfin) trfin = lj_ir_kptr(J, gcval(fin));
 943   } else if (tvisnil(fin)) {
 944     trfin = lj_ir_kptr(J, NULL);
 945   } else {
 946     lj_trace_err(J, LJ_TRERR_BADTYPE);
 947   }
 948   lj_ir_call(J, IRCALL_lj_cdata_setfin, trcd,
 949              trfin, lj_ir_kint(J, (int32_t)itype(fin)));
 950   J->needsnap = 1;
 951 }
 952 
 953 /* Record cdata allocation. */
 954 static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
 955 {
 956   CTState *cts = ctype_ctsG(J2G(J));
 957   CTSize sz;
 958   CTInfo info = lj_ctype_info(cts, id, &sz);
 959   CType *d = ctype_raw(cts, id);
 960   TRef trcd, trid = lj_ir_kint(J, id);
 961   cTValue *fin;
 962   /* Use special instruction to box pointer or 32/64 bit integer. */
 963   if (ctype_isptr(info) || (ctype_isinteger(info) && (sz == 4 || sz == 8))) {
 964     TRef sp = J->base[1] ? crec_ct_tv(J, d, 0, J->base[1], &rd->argv[1]) :
 965               ctype_isptr(info) ? lj_ir_kptr(J, NULL) :
 966               sz == 4 ? lj_ir_kint(J, 0) :
 967               (lj_needsplit(J), lj_ir_kint64(J, 0));
 968     J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, sp);
 969     return;
 970   } else {
 971     TRef trsz = TREF_NIL;
 972     if ((info & CTF_VLA)) {  /* Calculate VLA/VLS size at runtime. */
 973       CTSize sz0, sz1;
 974       if (!J->base[1] || J->base[2])
 975         lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init VLA/VLS. */
 976       trsz = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0,
 977                         J->base[1], &rd->argv[1]);
 978       sz0 = lj_ctype_vlsize(cts, d, 0);
 979       sz1 = lj_ctype_vlsize(cts, d, 1);
 980       trsz = emitir(IRTGI(IR_MULOV), trsz, lj_ir_kint(J, (int32_t)(sz1-sz0)));
 981       trsz = emitir(IRTGI(IR_ADDOV), trsz, lj_ir_kint(J, (int32_t)sz0));
 982       J->base[1] = 0;  /* Simplify logic below. */
 983     } else if (ctype_align(info) > CT_MEMALIGN) {
 984       trsz = lj_ir_kint(J, sz);
 985     }
 986     trcd = emitir(IRTG(IR_CNEW, IRT_CDATA), trid, trsz);
 987     if (sz > 128 || (info & CTF_VLA)) {
 988       TRef dp;
 989       CTSize align;
 990     special:  /* Only handle bulk zero-fill for large/VLA/VLS types. */
 991       if (J->base[1])
 992         lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init large/VLA/VLS types. */
 993       dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
 994       if (trsz == TREF_NIL) trsz = lj_ir_kint(J, sz);
 995       align = ctype_align(info);
 996       if (align < CT_MEMALIGN) align = CT_MEMALIGN;
 997       crec_fill(J, dp, trsz, lj_ir_kint(J, 0), (1u << align));
 998     } else if (J->base[1] && !J->base[2] &&
 999         !lj_cconv_multi_init(cts, d, &rd->argv[1])) {
1000       goto single_init;
1001     } else if (ctype_isarray(d->info)) {
1002       CType *dc = ctype_rawchild(cts, d);  /* Array element type. */
1003       CTSize ofs, esize = dc->size;
1004       TRef sp = 0;
1005       TValue tv;
1006       TValue *sval = &tv;
1007       MSize i;
1008       tv.u64 = 0;
1009       if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info)) ||
1010           esize * CREC_FILL_MAXUNROLL < sz)
1011         goto special;
1012       for (i = 1, ofs = 0; ofs < sz; ofs += esize) {
1013         TRef dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
1014                          lj_ir_kintp(J, ofs + sizeof(GCcdata)));
1015         if (J->base[i]) {
1016           sp = J->base[i];
1017           sval = &rd->argv[i];
1018           i++;
1019         } else if (i != 2) {
1020           sp = ctype_isnum(dc->info) ? lj_ir_kint(J, 0) : TREF_NIL;
1021         }
1022         crec_ct_tv(J, dc, dp, sp, sval);
1023       }
1024     } else if (ctype_isstruct(d->info)) {
1025       CTypeID fid = d->sib;
1026       MSize i = 1;
1027       while (fid) {
1028         CType *df = ctype_get(cts, fid);
1029         fid = df->sib;
1030         if (ctype_isfield(df->info)) {
1031           CType *dc;
1032           TRef sp, dp;
1033           TValue tv;
1034           TValue *sval = &tv;
1035           setintV(&tv, 0);
1036           if (!gcref(df->name)) continue;  /* Ignore unnamed fields. */
1037           dc = ctype_rawchild(cts, df);  /* Field type. */
1038           if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info) ||
1039                 ctype_isenum(dc->info)))
1040             lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init aggregates. */
1041           if (J->base[i]) {
1042             sp = J->base[i];
1043             sval = &rd->argv[i];
1044             i++;
1045           } else {
1046             sp = ctype_isptr(dc->info) ? TREF_NIL : lj_ir_kint(J, 0);
1047           }
1048           dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
1049                       lj_ir_kintp(J, df->size + sizeof(GCcdata)));
1050           crec_ct_tv(J, dc, dp, sp, sval);
1051         } else if (!ctype_isconstval(df->info)) {
1052           /* NYI: init bitfields and sub-structures. */
1053           lj_trace_err(J, LJ_TRERR_NYICONV);
1054         }
1055       }
1056     } else {
1057       TRef dp;
1058     single_init:
1059       dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
1060       if (J->base[1]) {
1061         crec_ct_tv(J, d, dp, J->base[1], &rd->argv[1]);
1062       } else {
1063         TValue tv;
1064         tv.u64 = 0;
1065         crec_ct_tv(J, d, dp, lj_ir_kint(J, 0), &tv);
1066       }
1067     }
1068   }
1069   J->base[0] = trcd;
1070   /* Handle __gc metamethod. */
1071   fin = lj_ctype_meta(cts, id, MM_gc);
1072   if (fin)
1073     crec_finalizer(J, trcd, 0, fin);
1074 }
1075 
1076 /* Record argument conversions. */
1077 static TRef crec_call_args(jit_State *J, RecordFFData *rd,
1078                            CTState *cts, CType *ct)
1079 {
1080   TRef args[CCI_NARGS_MAX];
1081   CTypeID fid;
1082   MSize i, n;
1083   TRef tr, *base;
1084   cTValue *o;
1085 #if LJ_TARGET_X86
1086 #if LJ_ABI_WIN
1087   TRef *arg0 = NULL, *arg1 = NULL;
1088 #endif
1089   int ngpr = 0;
1090   if (ctype_cconv(ct->info) == CTCC_THISCALL)
1091     ngpr = 1;
1092   else if (ctype_cconv(ct->info) == CTCC_FASTCALL)
1093     ngpr = 2;
1094 #endif
1095 
1096   /* Skip initial attributes. */
1097   fid = ct->sib;
1098   while (fid) {
1099     CType *ctf = ctype_get(cts, fid);
1100     if (!ctype_isattrib(ctf->info)) break;
1101     fid = ctf->sib;
1102   }
1103   args[0] = TREF_NIL;
1104   for (n = 0, base = J->base+1, o = rd->argv+1; *base; n++, base++, o++) {
1105     CTypeID did;
1106     CType *d;
1107 
1108     if (n >= CCI_NARGS_MAX)
1109       lj_trace_err(J, LJ_TRERR_NYICALL);
1110 
1111     if (fid) {  /* Get argument type from field. */
1112       CType *ctf = ctype_get(cts, fid);
1113       fid = ctf->sib;
1114       lua_assert(ctype_isfield(ctf->info));
1115       did = ctype_cid(ctf->info);
1116     } else {
1117       if (!(ct->info & CTF_VARARG))
1118         lj_trace_err(J, LJ_TRERR_NYICALL);  /* Too many arguments. */
1119       did = lj_ccall_ctid_vararg(cts, o);  /* Infer vararg type. */
1120     }
1121     d = ctype_raw(cts, did);
1122     if (!(ctype_isnum(d->info) || ctype_isptr(d->info) ||
1123           ctype_isenum(d->info)))
1124       lj_trace_err(J, LJ_TRERR_NYICALL);
1125     tr = crec_ct_tv(J, d, 0, *base, o);
1126     if (ctype_isinteger_or_bool(d->info)) {
1127       if (d->size < 4) {
1128         if ((d->info & CTF_UNSIGNED))
1129           tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_U8 : IRT_U16, 0);
1130         else
1131           tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_I8 : IRT_I16,IRCONV_SEXT);
1132       }
1133     } else if (LJ_SOFTFP32 && ctype_isfp(d->info) && d->size > 4) {
1134       lj_needsplit(J);
1135     }
1136 #if LJ_TARGET_X86
1137     /* 64 bit args must not end up in registers for fastcall/thiscall. */
1138 #if LJ_ABI_WIN
1139     if (!ctype_isfp(d->info)) {
1140       /* Sigh, the Windows/x86 ABI allows reordering across 64 bit args. */
1141       if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1142         if (ngpr) {
1143           arg0 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1144           if (ngpr) {
1145             arg1 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1146           }
1147         }
1148       } else {
1149         if (arg0) { *arg0 = tr; arg0 = NULL; n--; continue; }
1150         if (arg1) { *arg1 = tr; arg1 = NULL; n--; continue; }
1151         if (ngpr) ngpr--;
1152       }
1153     }
1154 #else
1155     if (!ctype_isfp(d->info) && ngpr) {
1156       if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1157         /* No reordering for other x86 ABIs. Simply add alignment args. */
1158         do { args[n++] = TREF_NIL; } while (--ngpr);
1159       } else {
1160         ngpr--;
1161       }
1162     }
1163 #endif
1164 #endif
1165     args[n] = tr;
1166   }
1167   tr = args[0];
1168   for (i = 1; i < n; i++)
1169     tr = emitir(IRT(IR_CARG, IRT_NIL), tr, args[i]);
1170   return tr;
1171 }
1172 
1173 /* Create a snapshot for the caller, simulating a 'false' return value. */
1174 static void crec_snap_caller(jit_State *J)
1175 {
1176   lua_State *L = J->L;
1177   TValue *base = L->base, *top = L->top;
1178   const BCIns *pc = J->pc;
1179   TRef ftr = J->base[-1-LJ_FR2];
1180   ptrdiff_t delta;
1181   if (!frame_islua(base-1) || J->framedepth <= 0)
1182     lj_trace_err(J, LJ_TRERR_NYICALL);
1183   J->pc = frame_pc(base-1); delta = 1+LJ_FR2+bc_a(J->pc[-1]);
1184   L->top = base; L->base = base - delta;
1185   J->base[-1-LJ_FR2] = TREF_FALSE;
1186   J->base -= delta; J->baseslot -= (BCReg)delta;
1187   J->maxslot = (BCReg)delta-LJ_FR2; J->framedepth--;
1188   lj_snap_add(J);
1189   L->base = base; L->top = top;
1190   J->framedepth++; J->maxslot = 1;
1191   J->base += delta; J->baseslot += (BCReg)delta;
1192   J->base[-1-LJ_FR2] = ftr; J->pc = pc;
1193 }
1194 
1195 /* Record function call. */
1196 static int crec_call(jit_State *J, RecordFFData *rd, GCcdata *cd)
1197 {
1198   CTState *cts = ctype_ctsG(J2G(J));
1199   CType *ct = ctype_raw(cts, cd->ctypeid);
1200   IRType tp = IRT_PTR;
1201   if (ctype_isptr(ct->info)) {
1202     tp = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
1203     ct = ctype_rawchild(cts, ct);
1204   }
1205   if (ctype_isfunc(ct->info)) {
1206     TRef func = emitir(IRT(IR_FLOAD, tp), J->base[0], IRFL_CDATA_PTR);
1207     CType *ctr = ctype_rawchild(cts, ct);
1208     IRType t = crec_ct2irt(cts, ctr);
1209     TRef tr;
1210     TValue tv;
1211     /* Check for blacklisted C functions that might call a callback. */
1212     setlightudV(&tv,
1213                 cdata_getptr(cdataptr(cd), (LJ_64 && tp == IRT_P64) ? 8 : 4));
1214     if (tvistrue(lj_tab_get(J->L, cts->miscmap, &tv)))
1215       lj_trace_err(J, LJ_TRERR_BLACKL);
1216     if (ctype_isvoid(ctr->info)) {
1217       t = IRT_NIL;
1218       rd->nres = 0;
1219     } else if (!(ctype_isnum(ctr->info) || ctype_isptr(ctr->info) ||
1220                  ctype_isenum(ctr->info)) || t == IRT_CDATA) {
1221       lj_trace_err(J, LJ_TRERR_NYICALL);
1222     }
1223     if ((ct->info & CTF_VARARG)
1224 #if LJ_TARGET_X86
1225         || ctype_cconv(ct->info) != CTCC_CDECL
1226 #endif
1227         )
1228       func = emitir(IRT(IR_CARG, IRT_NIL), func,
1229                     lj_ir_kint(J, ctype_typeid(cts, ct)));
1230     tr = emitir(IRT(IR_CALLXS, t), crec_call_args(J, rd, cts, ct), func);
1231     if (ctype_isbool(ctr->info)) {
1232       if (frame_islua(J->L->base-1) && bc_b(frame_pc(J->L->base-1)[-1]) == 1) {
1233         /* Don't check result if ignored. */
1234         tr = TREF_NIL;
1235       } else {
1236         crec_snap_caller(J);
1237 #if LJ_TARGET_X86ORX64
1238         /* Note: only the x86/x64 backend supports U8 and only for EQ(tr, 0). */
1239         lj_ir_set(J, IRTG(IR_NE, IRT_U8), tr, lj_ir_kint(J, 0));
1240 #else
1241         lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
1242 #endif
1243         J->postproc = LJ_POST_FIXGUARDSNAP;
1244         tr = TREF_TRUE;
1245       }
1246     } else if (t == IRT_PTR || (LJ_64 && t == IRT_P32) ||
1247                t == IRT_I64 || t == IRT_U64 || ctype_isenum(ctr->info)) {
1248       TRef trid = lj_ir_kint(J, ctype_cid(ct->info));
1249       tr = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, tr);
1250       if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1251     } else if (t == IRT_FLOAT || t == IRT_U32) {
1252       tr = emitconv(tr, IRT_NUM, t, 0);
1253     } else if (t == IRT_I8 || t == IRT_I16) {
1254       tr = emitconv(tr, IRT_INT, t, IRCONV_SEXT);
1255     } else if (t == IRT_U8 || t == IRT_U16) {
1256       tr = emitconv(tr, IRT_INT, t, 0);
1257     }
1258     J->base[0] = tr;
1259     J->needsnap = 1;
1260     return 1;
1261   }
1262   return 0;
1263 }
1264 
1265 void LJ_FASTCALL recff_cdata_call(jit_State *J, RecordFFData *rd)
1266 {
1267   CTState *cts = ctype_ctsG(J2G(J));
1268   GCcdata *cd = argv2cdata(J, J->base[0], &rd->argv[0]);
1269   CTypeID id = cd->ctypeid;
1270   CType *ct;
1271   cTValue *tv;
1272   MMS mm = MM_call;
1273   if (id == CTID_CTYPEID) {
1274     id = crec_constructor(J, cd, J->base[0]);
1275     mm = MM_new;
1276   } else if (crec_call(J, rd, cd)) {
1277     return;
1278   }
1279   /* Record ctype __call/__new metamethod. */
1280   ct = ctype_raw(cts, id);
1281   tv = lj_ctype_meta(cts, ctype_isptr(ct->info) ? ctype_cid(ct->info) : id, mm);
1282   if (tv) {
1283     if (tvisfunc(tv)) {
1284       crec_tailcall(J, rd, tv);
1285       return;
1286     }
1287   } else if (mm == MM_new) {
1288     crec_alloc(J, rd, id);
1289     return;
1290   }
1291   /* No metamethod or NYI: non-function metamethods. */
1292   lj_trace_err(J, LJ_TRERR_BADTYPE);
1293 }
1294 
1295 static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
1296 {
1297   if (sp[0] && sp[1] && ctype_isnum(s[0]->info) && ctype_isnum(s[1]->info)) {
1298     IRType dt;
1299     CTypeID id;
1300     TRef tr;
1301     MSize i;
1302     IROp op;
1303     lj_needsplit(J);
1304     if (((s[0]->info & CTF_UNSIGNED) && s[0]->size == 8) ||
1305         ((s[1]->info & CTF_UNSIGNED) && s[1]->size == 8)) {
1306       dt = IRT_U64; id = CTID_UINT64;
1307     } else {
1308       dt = IRT_I64; id = CTID_INT64;
1309       if (mm < MM_add &&
1310           !((s[0]->info | s[1]->info) & CTF_FP) &&
1311           s[0]->size == 4 && s[1]->size == 4) {  /* Try to narrow comparison. */
1312         if (!((s[0]->info ^ s[1]->info) & CTF_UNSIGNED) ||
1313             (tref_isk(sp[1]) && IR(tref_ref(sp[1]))->i >= 0)) {
1314           dt = (s[0]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1315           goto comp;
1316         } else if (tref_isk(sp[0]) && IR(tref_ref(sp[0]))->i >= 0) {
1317           dt = (s[1]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1318           goto comp;
1319         }
1320       }
1321     }
1322     for (i = 0; i < 2; i++) {
1323       IRType st = tref_type(sp[i]);
1324       if (st == IRT_NUM || st == IRT_FLOAT)
1325         sp[i] = emitconv(sp[i], dt, st, IRCONV_ANY);
1326       else if (!(st == IRT_I64 || st == IRT_U64))
1327         sp[i] = emitconv(sp[i], dt, IRT_INT,
1328                          (s[i]->info & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
1329     }
1330     if (mm < MM_add) {
1331     comp:
1332       /* Assume true comparison. Fixup and emit pending guard later. */
1333       if (mm == MM_eq) {
1334         op = IR_EQ;
1335       } else {
1336         op = mm == MM_lt ? IR_LT : IR_LE;
1337         if (dt == IRT_U32 || dt == IRT_U64)
1338           op += (IR_ULT-IR_LT);
1339       }
1340       lj_ir_set(J, IRTG(op, dt), sp[0], sp[1]);
1341       J->postproc = LJ_POST_FIXGUARD;
1342       return TREF_TRUE;
1343     } else {
1344       tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, dt), sp[0], sp[1]);
1345     }
1346     return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1347   }
1348   return 0;
1349 }
1350 
1351 static TRef crec_arith_ptr(jit_State *J, TRef *sp, CType **s, MMS mm)
1352 {
1353   CTState *cts = ctype_ctsG(J2G(J));
1354   CType *ctp = s[0];
1355   if (!(sp[0] && sp[1])) return 0;
1356   if (ctype_isptr(ctp->info) || ctype_isrefarray(ctp->info)) {
1357     if ((mm == MM_sub || mm == MM_eq || mm == MM_lt || mm == MM_le) &&
1358         (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1359       if (mm == MM_sub) {  /* Pointer difference. */
1360         TRef tr;
1361         CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1362         if (sz == 0 || (sz & (sz-1)) != 0)
1363           return 0;  /* NYI: integer division. */
1364         tr = emitir(IRT(IR_SUB, IRT_INTP), sp[0], sp[1]);
1365         tr = emitir(IRT(IR_BSAR, IRT_INTP), tr, lj_ir_kint(J, lj_fls(sz)));
1366 #if LJ_64
1367         tr = emitconv(tr, IRT_NUM, IRT_INTP, 0);
1368 #endif
1369         return tr;
1370       } else {  /* Pointer comparison (unsigned). */
1371         /* Assume true comparison. Fixup and emit pending guard later. */
1372         IROp op = mm == MM_eq ? IR_EQ : mm == MM_lt ? IR_ULT : IR_ULE;
1373         lj_ir_set(J, IRTG(op, IRT_PTR), sp[0], sp[1]);
1374         J->postproc = LJ_POST_FIXGUARD;
1375         return TREF_TRUE;
1376       }
1377     }
1378     if (!((mm == MM_add || mm == MM_sub) && ctype_isnum(s[1]->info)))
1379       return 0;
1380   } else if (mm == MM_add && ctype_isnum(ctp->info) &&
1381              (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1382     TRef tr = sp[0]; sp[0] = sp[1]; sp[1] = tr;  /* Swap pointer and index. */
1383     ctp = s[1];
1384   } else {
1385     return 0;
1386   }
1387   {
1388     TRef tr = sp[1];
1389     IRType t = tref_type(tr);
1390     CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1391     CTypeID id;
1392 #if LJ_64
1393     if (t == IRT_NUM || t == IRT_FLOAT)
1394       tr = emitconv(tr, IRT_INTP, t, IRCONV_ANY);
1395     else if (!(t == IRT_I64 || t == IRT_U64))
1396       tr = emitconv(tr, IRT_INTP, IRT_INT,
1397                     ((t - IRT_I8) & 1) ? 0 : IRCONV_SEXT);
1398 #else
1399     if (!tref_typerange(sp[1], IRT_I8, IRT_U32)) {
1400       tr = emitconv(tr, IRT_INTP, t,
1401                     (t == IRT_NUM || t == IRT_FLOAT) ? IRCONV_ANY : 0);
1402     }
1403 #endif
1404     tr = emitir(IRT(IR_MUL, IRT_INTP), tr, lj_ir_kintp(J, sz));
1405     tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, IRT_PTR), sp[0], tr);
1406     id = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(ctp->info)),
1407                          CTSIZE_PTR);
1408     return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1409   }
1410 }
1411 
1412 /* Record ctype arithmetic metamethods. */
1413 static TRef crec_arith_meta(jit_State *J, TRef *sp, CType **s, CTState *cts,
1414                             RecordFFData *rd)
1415 {
1416   cTValue *tv = NULL;
1417   if (J->base[0]) {
1418     if (tviscdata(&rd->argv[0])) {
1419       CTypeID id = argv2cdata(J, J->base[0], &rd->argv[0])->ctypeid;
1420       CType *ct = ctype_raw(cts, id);
1421       if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1422       tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1423     }
1424     if (!tv && J->base[1] && tviscdata(&rd->argv[1])) {
1425       CTypeID id = argv2cdata(J, J->base[1], &rd->argv[1])->ctypeid;
1426       CType *ct = ctype_raw(cts, id);
1427       if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1428       tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1429     }
1430   }
1431   if (tv) {
1432     if (tvisfunc(tv)) {
1433       crec_tailcall(J, rd, tv);
1434       return 0;
1435     }  /* NYI: non-function metamethods. */
1436   } else if ((MMS)rd->data == MM_eq) {  /* Fallback cdata pointer comparison. */
1437     if (sp[0] && sp[1] && ctype_isnum(s[0]->info) == ctype_isnum(s[1]->info)) {
1438       /* Assume true comparison. Fixup and emit pending guard later. */
1439       lj_ir_set(J, IRTG(IR_EQ, IRT_PTR), sp[0], sp[1]);
1440       J->postproc = LJ_POST_FIXGUARD;
1441       return TREF_TRUE;
1442     } else {
1443       return TREF_FALSE;
1444     }
1445   }
1446   lj_trace_err(J, LJ_TRERR_BADTYPE);
1447   return 0;
1448 }
1449 
1450 void LJ_FASTCALL recff_cdata_arith(jit_State *J, RecordFFData *rd)
1451 {
1452   CTState *cts = ctype_ctsG(J2G(J));
1453   TRef sp[2];
1454   CType *s[2];
1455   MSize i;
1456   for (i = 0; i < 2; i++) {
1457     TRef tr = J->base[i];
1458     CType *ct = ctype_get(cts, CTID_DOUBLE);
1459     if (!tr) {
1460       lj_trace_err(J, LJ_TRERR_BADTYPE);
1461     } else if (tref_iscdata(tr)) {
1462       CTypeID id = argv2cdata(J, tr, &rd->argv[i])->ctypeid;
1463       IRType t;
1464       ct = ctype_raw(cts, id);
1465       t = crec_ct2irt(cts, ct);
1466       if (ctype_isptr(ct->info)) {  /* Resolve pointer or reference. */
1467         tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_PTR);
1468         if (ctype_isref(ct->info)) {
1469           ct = ctype_rawchild(cts, ct);
1470           t = crec_ct2irt(cts, ct);
1471         }
1472       } else if (t == IRT_I64 || t == IRT_U64) {
1473         tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT64);
1474         lj_needsplit(J);
1475         goto ok;
1476       } else if (t == IRT_INT || t == IRT_U32) {
1477         tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT);
1478         if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1479         goto ok;
1480       } else if (ctype_isfunc(ct->info)) {
1481         tr = emitir(IRT(IR_FLOAD, IRT_PTR), tr, IRFL_CDATA_PTR);
1482         ct = ctype_get(cts,
1483           lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR));
1484         goto ok;
1485       } else {
1486         tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCcdata)));
1487       }
1488       if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1489       if (ctype_isnum(ct->info)) {
1490         if (t == IRT_CDATA) {
1491           tr = 0;
1492         } else {
1493           if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1494           tr = emitir(IRT(IR_XLOAD, t), tr, 0);
1495         }
1496       }
1497     } else if (tref_isnil(tr)) {
1498       tr = lj_ir_kptr(J, NULL);
1499       ct = ctype_get(cts, CTID_P_VOID);
1500     } else if (tref_isinteger(tr)) {
1501       ct = ctype_get(cts, CTID_INT32);
1502     } else if (tref_isstr(tr)) {
1503       TRef tr2 = J->base[1-i];
1504       CTypeID id = argv2cdata(J, tr2, &rd->argv[1-i])->ctypeid;
1505       ct = ctype_raw(cts, id);
1506       if (ctype_isenum(ct->info)) {  /* Match string against enum constant. */
1507         GCstr *str = strV(&rd->argv[i]);
1508         CTSize ofs;
1509         CType *cct = lj_ctype_getfield(cts, ct, str, &ofs);
1510         if (cct && ctype_isconstval(cct->info)) {
1511           /* Specialize to the name of the enum constant. */
1512           emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, str));
1513           ct = ctype_child(cts, cct);
1514           tr = lj_ir_kint(J, (int32_t)ofs);
1515         } else {  /* Interpreter will throw or return false. */
1516           ct = ctype_get(cts, CTID_P_VOID);
1517         }
1518       } else if (ctype_isptr(ct->info)) {
1519         tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCstr)));
1520       } else {
1521         ct = ctype_get(cts, CTID_P_VOID);
1522       }
1523     } else if (!tref_isnum(tr)) {
1524       tr = 0;
1525       ct = ctype_get(cts, CTID_P_VOID);
1526     }
1527   ok:
1528     s[i] = ct;
1529     sp[i] = tr;
1530   }
1531   {
1532     TRef tr;
1533     if (!(tr = crec_arith_int64(J, sp, s, (MMS)rd->data)) &&
1534         !(tr = crec_arith_ptr(J, sp, s, (MMS)rd->data)) &&
1535         !(tr = crec_arith_meta(J, sp, s, cts, rd)))
1536       return;
1537     J->base[0] = tr;
1538     /* Fixup cdata comparisons, too. Avoids some cdata escapes. */
1539     if (J->postproc == LJ_POST_FIXGUARD && frame_iscont(J->L->base-1) &&
1540         !irt_isguard(J->guardemit)) {
1541       const BCIns *pc = frame_contpc(J->L->base-1) - 1;
1542       if (bc_op(*pc) <= BC_ISNEP) {
1543         J2G(J)->tmptv.u64 = (uint64_t)(uintptr_t)pc;
1544         J->postproc = LJ_POST_FIXCOMP;
1545       }
1546     }
1547   }
1548 }
1549 
1550 /* -- C library namespace metamethods ------------------------------------- */
1551 
1552 void LJ_FASTCALL recff_clib_index(jit_State *J, RecordFFData *rd)
1553 {
1554   CTState *cts = ctype_ctsG(J2G(J));
1555   if (tref_isudata(J->base[0]) && tref_isstr(J->base[1]) &&
1556       udataV(&rd->argv[0])->udtype == UDTYPE_FFI_CLIB) {
1557     CLibrary *cl = (CLibrary *)uddata(udataV(&rd->argv[0]));
1558     GCstr *name = strV(&rd->argv[1]);
1559     CType *ct;
1560     CTypeID id = lj_ctype_getname(cts, &ct, name, CLNS_INDEX);
1561     cTValue *tv = lj_tab_getstr(cl->cache, name);
1562     rd->nres = rd->data;
1563     if (id && tv && !tvisnil(tv)) {
1564       /* Specialize to the symbol name and make the result a constant. */
1565       emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, name));
1566       if (ctype_isconstval(ct->info)) {
1567         if (ct->size >= 0x80000000u &&
1568             (ctype_child(cts, ct)->info & CTF_UNSIGNED))
1569           J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)ct->size);
1570         else
1571           J->base[0] = lj_ir_kint(J, (int32_t)ct->size);
1572       } else if (ctype_isextern(ct->info)) {
1573         CTypeID sid = ctype_cid(ct->info);
1574         void *sp = *(void **)cdataptr(cdataV(tv));
1575         TRef ptr;
1576         ct = ctype_raw(cts, sid);
1577         if (LJ_64 && !checkptr32(sp))
1578           ptr = lj_ir_kintp(J, (uintptr_t)sp);
1579         else
1580           ptr = lj_ir_kptr(J, sp);
1581         if (rd->data) {
1582           J->base[0] = crec_tv_ct(J, ct, sid, ptr);
1583         } else {
1584           J->needsnap = 1;
1585           crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
1586         }
1587       } else {
1588         J->base[0] = lj_ir_kgc(J, obj2gco(cdataV(tv)), IRT_CDATA);
1589       }
1590     } else {
1591       lj_trace_err(J, LJ_TRERR_NOCACHE);
1592     }
1593   }  /* else: interpreter will throw. */
1594 }
1595 
1596 /* -- FFI library functions ----------------------------------------------- */
1597 
1598 static TRef crec_toint(jit_State *J, CTState *cts, TRef sp, TValue *sval)
1599 {
1600   return crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, sp, sval);
1601 }
1602 
1603 void LJ_FASTCALL recff_ffi_new(jit_State *J, RecordFFData *rd)
1604 {
1605   crec_alloc(J, rd, argv2ctype(J, J->base[0], &rd->argv[0]));
1606 }
1607 
1608 void LJ_FASTCALL recff_ffi_errno(jit_State *J, RecordFFData *rd)
1609 {
1610   UNUSED(rd);
1611   if (J->base[0])
1612     lj_trace_err(J, LJ_TRERR_NYICALL);
1613   J->base[0] = lj_ir_call(J, IRCALL_lj_vm_errno);
1614 }
1615 
1616 void LJ_FASTCALL recff_ffi_string(jit_State *J, RecordFFData *rd)
1617 {
1618   CTState *cts = ctype_ctsG(J2G(J));
1619   TRef tr = J->base[0];
1620   if (tr) {
1621     TRef trlen = J->base[1];
1622     if (!tref_isnil(trlen)) {
1623       trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1624       tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, tr, &rd->argv[0]);
1625     } else {
1626       tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CCHAR), 0, tr, &rd->argv[0]);
1627       trlen = lj_ir_call(J, IRCALL_strlen, tr);
1628     }
1629     J->base[0] = emitir(IRT(IR_XSNEW, IRT_STR), tr, trlen);
1630   }  /* else: interpreter will throw. */
1631 }
1632 
1633 void LJ_FASTCALL recff_ffi_copy(jit_State *J, RecordFFData *rd)
1634 {
1635   CTState *cts = ctype_ctsG(J2G(J));
1636   TRef trdst = J->base[0], trsrc = J->base[1], trlen = J->base[2];
1637   if (trdst && trsrc && (trlen || tref_isstr(trsrc))) {
1638     trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1639     trsrc = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, trsrc, &rd->argv[1]);
1640     if (trlen) {
1641       trlen = crec_toint(J, cts, trlen, &rd->argv[2]);
1642     } else {
1643       trlen = emitir(IRTI(IR_FLOAD), J->base[1], IRFL_STR_LEN);
1644       trlen = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1));
1645     }
1646     rd->nres = 0;
1647     crec_copy(J, trdst, trsrc, trlen, NULL);
1648   }  /* else: interpreter will throw. */
1649 }
1650 
1651 void LJ_FASTCALL recff_ffi_fill(jit_State *J, RecordFFData *rd)
1652 {
1653   CTState *cts = ctype_ctsG(J2G(J));
1654   TRef trdst = J->base[0], trlen = J->base[1], trfill = J->base[2];
1655   if (trdst && trlen) {
1656     CTSize step = 1;
1657     if (tviscdata(&rd->argv[0])) {  /* Get alignment of original destination. */
1658       CTSize sz;
1659       CType *ct = ctype_raw(cts, cdataV(&rd->argv[0])->ctypeid);
1660       if (ctype_isptr(ct->info))
1661         ct = ctype_rawchild(cts, ct);
1662       step = (1u<<ctype_align(lj_ctype_info(cts, ctype_typeid(cts, ct), &sz)));
1663     }
1664     trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1665     trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1666     if (trfill)
1667       trfill = crec_toint(J, cts, trfill, &rd->argv[2]);
1668     else
1669       trfill = lj_ir_kint(J, 0);
1670     rd->nres = 0;
1671     crec_fill(J, trdst, trlen, trfill, step);
1672   }  /* else: interpreter will throw. */
1673 }
1674 
1675 void LJ_FASTCALL recff_ffi_typeof(jit_State *J, RecordFFData *rd)
1676 {
1677   if (tref_iscdata(J->base[0])) {
1678     TRef trid = lj_ir_kint(J, argv2ctype(J, J->base[0], &rd->argv[0]));
1679     J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA),
1680                         lj_ir_kint(J, CTID_CTYPEID), trid);
1681   } else {
1682     setfuncV(J->L, &J->errinfo, J->fn);
1683     lj_trace_err_info(J, LJ_TRERR_NYIFFU);
1684   }
1685 }
1686 
1687 void LJ_FASTCALL recff_ffi_istype(jit_State *J, RecordFFData *rd)
1688 {
1689   argv2ctype(J, J->base[0], &rd->argv[0]);
1690   if (tref_iscdata(J->base[1])) {
1691     argv2ctype(J, J->base[1], &rd->argv[1]);
1692     J->postproc = LJ_POST_FIXBOOL;
1693     J->base[0] = TREF_TRUE;
1694   } else {
1695     J->base[0] = TREF_FALSE;
1696   }
1697 }
1698 
1699 void LJ_FASTCALL recff_ffi_abi(jit_State *J, RecordFFData *rd)
1700 {
1701   if (tref_isstr(J->base[0])) {
1702     /* Specialize to the ABI string to make the boolean result a constant. */
1703     emitir(IRTG(IR_EQ, IRT_STR), J->base[0], lj_ir_kstr(J, strV(&rd->argv[0])));
1704     J->postproc = LJ_POST_FIXBOOL;
1705     J->base[0] = TREF_TRUE;
1706   } else {
1707     lj_trace_err(J, LJ_TRERR_BADTYPE);
1708   }
1709 }
1710 
1711 /* Record ffi.sizeof(), ffi.alignof(), ffi.offsetof(). */
1712 void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd)
1713 {
1714   CTypeID id = argv2ctype(J, J->base[0], &rd->argv[0]);
1715   if (rd->data == FF_ffi_sizeof) {
1716     CType *ct = lj_ctype_rawref(ctype_ctsG(J2G(J)), id);
1717     if (ctype_isvltype(ct->info))
1718       lj_trace_err(J, LJ_TRERR_BADTYPE);
1719   } else if (rd->data == FF_ffi_offsetof) {  /* Specialize to the field name. */
1720     if (!tref_isstr(J->base[1]))
1721       lj_trace_err(J, LJ_TRERR_BADTYPE);
1722     emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
1723     rd->nres = 3;  /* Just in case. */
1724   }
1725   J->postproc = LJ_POST_FIXCONST;
1726   J->base[0] = J->base[1] = J->base[2] = TREF_NIL;
1727 }
1728 
1729 void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd)
1730 {
1731   argv2cdata(J, J->base[0], &rd->argv[0]);
1732   if (!J->base[1])
1733     lj_trace_err(J, LJ_TRERR_BADTYPE);
1734   crec_finalizer(J, J->base[0], J->base[1], &rd->argv[1]);
1735 }
1736 
1737 /* -- 64 bit bit.* library functions -------------------------------------- */
1738 
1739 /* Determine bit operation type from argument type. */
1740 static CTypeID crec_bit64_type(CTState *cts, cTValue *tv)
1741 {
1742   if (tviscdata(tv)) {
1743     CType *ct = lj_ctype_rawref(cts, cdataV(tv)->ctypeid);
1744     if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1745     if ((ct->info & (CTMASK_NUM|CTF_BOOL|CTF_FP|CTF_UNSIGNED)) ==
1746         CTINFO(CT_NUM, CTF_UNSIGNED) && ct->size == 8)
1747       return CTID_UINT64;  /* Use uint64_t, since it has the highest rank. */
1748     return CTID_INT64;  /* Otherwise use int64_t. */
1749   }
1750   return 0;  /* Use regular 32 bit ops. */
1751 }
1752 
1753 void LJ_FASTCALL recff_bit64_tobit(jit_State *J, RecordFFData *rd)
1754 {
1755   CTState *cts = ctype_ctsG(J2G(J));
1756   TRef tr = crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
1757                        J->base[0], &rd->argv[0]);
1758   if (!tref_isinteger(tr))
1759     tr = emitconv(tr, IRT_INT, tref_type(tr), 0);
1760   J->base[0] = tr;
1761 }
1762 
1763 int LJ_FASTCALL recff_bit64_unary(jit_State *J, RecordFFData *rd)
1764 {
1765   CTState *cts = ctype_ctsG(J2G(J));
1766   CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
1767   if (id) {
1768     TRef tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1769     tr = emitir(IRT(rd->data, id-CTID_INT64+IRT_I64), tr, 0);
1770     J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1771     return 1;
1772   }
1773   return 0;
1774 }
1775 
1776 int LJ_FASTCALL recff_bit64_nary(jit_State *J, RecordFFData *rd)
1777 {
1778   CTState *cts = ctype_ctsG(J2G(J));
1779   CTypeID id = 0;
1780   MSize i;
1781   for (i = 0; J->base[i] != 0; i++) {
1782     CTypeID aid = crec_bit64_type(cts, &rd->argv[i]);
1783     if (id < aid) id = aid;  /* Determine highest type rank of all arguments. */
1784   }
1785   if (id) {
1786     CType *ct = ctype_get(cts, id);
1787     uint32_t ot = IRT(rd->data, id-CTID_INT64+IRT_I64);
1788     TRef tr = crec_ct_tv(J, ct, 0, J->base[0], &rd->argv[0]);
1789     for (i = 1; J->base[i] != 0; i++) {
1790       TRef tr2 = crec_ct_tv(J, ct, 0, J->base[i], &rd->argv[i]);
1791       tr = emitir(ot, tr, tr2);
1792     }
1793     J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1794     return 1;
1795   }
1796   return 0;
1797 }
1798 
1799 int LJ_FASTCALL recff_bit64_shift(jit_State *J, RecordFFData *rd)
1800 {
1801   CTState *cts = ctype_ctsG(J2G(J));
1802   CTypeID id;
1803   TRef tsh = 0;
1804   if (J->base[0] && tref_iscdata(J->base[1])) {
1805     tsh = crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
1806                      J->base[1], &rd->argv[1]);
1807     if (!tref_isinteger(tsh))
1808       tsh = emitconv(tsh, IRT_INT, tref_type(tsh), 0);
1809     J->base[1] = tsh;
1810   }
1811   id = crec_bit64_type(cts, &rd->argv[0]);
1812   if (id) {
1813     TRef tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1814     uint32_t op = rd->data;
1815     if (!tsh) tsh = lj_opt_narrow_tobit(J, J->base[1]);
1816     if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
1817         !tref_isk(tsh))
1818       tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 63));
1819 #ifdef LJ_TARGET_UNIFYROT
1820       if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
1821         op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
1822         tsh = emitir(IRTI(IR_NEG), tsh, tsh);
1823       }
1824 #endif
1825     tr = emitir(IRT(op, id-CTID_INT64+IRT_I64), tr, tsh);
1826     J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1827     return 1;
1828   }
1829   return 0;
1830 }
1831 
1832 TRef recff_bit64_tohex(jit_State *J, RecordFFData *rd, TRef hdr)
1833 {
1834   CTState *cts = ctype_ctsG(J2G(J));
1835   CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
1836   TRef tr, trsf = J->base[1];
1837   SFormat sf = (STRFMT_UINT|STRFMT_T_HEX);
1838   int32_t n;
1839   if (trsf) {
1840     CTypeID id2 = 0;
1841     n = (int32_t)lj_carith_check64(J->L, 2, &id2);
1842     if (id2)
1843       trsf = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, trsf, &rd->argv[1]);
1844     else
1845       trsf = lj_opt_narrow_tobit(J, trsf);
1846     emitir(IRTGI(IR_EQ), trsf, lj_ir_kint(J, n));  /* Specialize to n. */
1847   } else {
1848     n = id ? 16 : 8;
1849   }
1850   if (n < 0) { n = -n; sf |= STRFMT_F_UPPER; }
1851   sf |= ((SFormat)((n+1)&255) << STRFMT_SH_PREC);
1852   if (id) {
1853     tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1854     if (n < 16)
1855       tr = emitir(IRT(IR_BAND, IRT_U64), tr,
1856                   lj_ir_kint64(J, ((uint64_t)1 << 4*n)-1));
1857   } else {
1858     tr = lj_opt_narrow_tobit(J, J->base[0]);
1859     if (n < 8)
1860       tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << 4*n)-1)));
1861     tr = emitconv(tr, IRT_U64, IRT_INT, 0);  /* No sign-extension. */
1862     lj_needsplit(J);
1863   }
1864   return lj_ir_call(J, IRCALL_lj_strfmt_putfxint, hdr, lj_ir_kint(J, sf), tr);
1865 }
1866 
1867 /* -- Miscellaneous library functions ------------------------------------- */
1868 
1869 void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd)
1870 {
1871   CTState *cts = ctype_ctsG(J2G(J));
1872   CType *d, *ct = lj_ctype_rawref(cts, cdataV(&rd->argv[0])->ctypeid);
1873   if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1874   if (ctype_isnum(ct->info) || ctype_iscomplex(ct->info)) {
1875     if (ctype_isinteger_or_bool(ct->info) && ct->size <= 4 &&
1876         !(ct->size == 4 && (ct->info & CTF_UNSIGNED)))
1877       d = ctype_get(cts, CTID_INT32);
1878     else
1879       d = ctype_get(cts, CTID_DOUBLE);
1880     J->base[0] = crec_ct_tv(J, d, 0, J->base[0], &rd->argv[0]);
1881   } else {
1882     J->base[0] = TREF_NIL;
1883   }
1884 }
1885 
1886 #undef IR
1887 #undef emitir
1888 #undef emitconv
1889 
1890 #endif

/* [<][>][^][v][top][bottom][index][help] */