root/lj_crecord.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. argv2cdata
  2. crec_constructor
  3. argv2ctype
  4. crec_ct2irt
  5. crec_copy_struct
  6. crec_copy_unroll
  7. crec_copy_emit
  8. crec_copy
  9. crec_fill_unroll
  10. crec_fill_emit
  11. crec_fill
  12. crec_isnonzero
  13. crec_ct_ct
  14. crec_tv_ct
  15. crec_ct_tv
  16. crec_reassoc_ofs
  17. crec_index_meta
  18. recff_cdata_index
  19. crec_finalizer
  20. crec_alloc
  21. crec_call_args
  22. crec_snap_caller
  23. crec_call
  24. recff_cdata_call
  25. crec_arith_int64
  26. crec_arith_ptr
  27. crec_arith_meta
  28. recff_cdata_arith
  29. recff_clib_index
  30. crec_toint
  31. recff_ffi_new
  32. recff_ffi_errno
  33. recff_ffi_string
  34. recff_ffi_copy
  35. recff_ffi_fill
  36. recff_ffi_typeof
  37. recff_ffi_istype
  38. recff_ffi_abi
  39. recff_ffi_xof
  40. recff_ffi_gc
  41. lj_crecord_tonumber

   1 /*
   2 ** Trace recorder for C data operations.
   3 ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5 
   6 #define lj_ffrecord_c
   7 #define LUA_CORE
   8 
   9 #include "lj_obj.h"
  10 
  11 #if LJ_HASJIT && LJ_HASFFI
  12 
  13 #include "lj_err.h"
  14 #include "lj_str.h"
  15 #include "lj_tab.h"
  16 #include "lj_frame.h"
  17 #include "lj_ctype.h"
  18 #include "lj_cdata.h"
  19 #include "lj_cparse.h"
  20 #include "lj_cconv.h"
  21 #include "lj_clib.h"
  22 #include "lj_ccall.h"
  23 #include "lj_ff.h"
  24 #include "lj_ir.h"
  25 #include "lj_jit.h"
  26 #include "lj_ircall.h"
  27 #include "lj_iropt.h"
  28 #include "lj_trace.h"
  29 #include "lj_record.h"
  30 #include "lj_ffrecord.h"
  31 #include "lj_snap.h"
  32 #include "lj_crecord.h"
  33 #include "lj_dispatch.h"
  34 
  35 /* Some local macros to save typing. Undef'd at the end. */
  36 #define IR(ref)                 (&J->cur.ir[(ref)])
  37 
  38 /* Pass IR on to next optimization in chain (FOLD). */
  39 #define emitir(ot, a, b)        (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
  40 
  41 #define emitconv(a, dt, st, flags) \
  42   emitir(IRT(IR_CONV, (dt)), (a), (st)|((dt) << 5)|(flags))
  43 
  44 /* -- C type checks ------------------------------------------------------- */
  45 
  46 static GCcdata *argv2cdata(jit_State *J, TRef tr, cTValue *o)
  47 {
  48   GCcdata *cd;
  49   TRef trtypeid;
  50   if (!tref_iscdata(tr))
  51     lj_trace_err(J, LJ_TRERR_BADTYPE);
  52   cd = cdataV(o);
  53   /* Specialize to the CTypeID. */
  54   trtypeid = emitir(IRT(IR_FLOAD, IRT_U16), tr, IRFL_CDATA_CTYPEID);
  55   emitir(IRTG(IR_EQ, IRT_INT), trtypeid, lj_ir_kint(J, (int32_t)cd->ctypeid));
  56   return cd;
  57 }
  58 
  59 /* Specialize to the CTypeID held by a cdata constructor. */
  60 static CTypeID crec_constructor(jit_State *J, GCcdata *cd, TRef tr)
  61 {
  62   CTypeID id;
  63   lua_assert(tref_iscdata(tr) && cd->ctypeid == CTID_CTYPEID);
  64   id = *(CTypeID *)cdataptr(cd);
  65   tr = emitir(IRT(IR_FLOAD, IRT_INT), tr, IRFL_CDATA_INT);
  66   emitir(IRTG(IR_EQ, IRT_INT), tr, lj_ir_kint(J, (int32_t)id));
  67   return id;
  68 }
  69 
  70 static CTypeID argv2ctype(jit_State *J, TRef tr, cTValue *o)
  71 {
  72   if (tref_isstr(tr)) {
  73     GCstr *s = strV(o);
  74     CPState cp;
  75     CTypeID oldtop;
  76     /* Specialize to the string containing the C type declaration. */
  77     emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, s));
  78     cp.L = J->L;
  79     cp.cts = ctype_ctsG(J2G(J));
  80     oldtop = cp.cts->top;
  81     cp.srcname = strdata(s);
  82     cp.p = strdata(s);
  83     cp.param = NULL;
  84     cp.mode = CPARSE_MODE_ABSTRACT|CPARSE_MODE_NOIMPLICIT;
  85     if (lj_cparse(&cp) || cp.cts->top > oldtop)  /* Avoid new struct defs. */
  86       lj_trace_err(J, LJ_TRERR_BADTYPE);
  87     return cp.val.id;
  88   } else {
  89     GCcdata *cd = argv2cdata(J, tr, o);
  90     return cd->ctypeid == CTID_CTYPEID ? crec_constructor(J, cd, tr) :
  91                                         cd->ctypeid;
  92   }
  93 }
  94 
  95 /* Convert CType to IRType (if possible). */
  96 static IRType crec_ct2irt(CTState *cts, CType *ct)
  97 {
  98   if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
  99   if (LJ_LIKELY(ctype_isnum(ct->info))) {
 100     if ((ct->info & CTF_FP)) {
 101       if (ct->size == sizeof(double))
 102         return IRT_NUM;
 103       else if (ct->size == sizeof(float))
 104         return IRT_FLOAT;
 105     } else {
 106       uint32_t b = lj_fls(ct->size);
 107       if (b <= 3)
 108         return IRT_I8 + 2*b + ((ct->info & CTF_UNSIGNED) ? 1 : 0);
 109     }
 110   } else if (ctype_isptr(ct->info)) {
 111     return (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
 112   } else if (ctype_iscomplex(ct->info)) {
 113     if (ct->size == 2*sizeof(double))
 114       return IRT_NUM;
 115     else if (ct->size == 2*sizeof(float))
 116       return IRT_FLOAT;
 117   }
 118   return IRT_CDATA;
 119 }
 120 
 121 /* -- Optimized memory fill and copy -------------------------------------- */
 122 
 123 /* Maximum length and unroll of inlined copy/fill. */
 124 #define CREC_COPY_MAXUNROLL             16
 125 #define CREC_COPY_MAXLEN                128
 126 
 127 #define CREC_FILL_MAXUNROLL             16
 128 
 129 /* Number of windowed registers used for optimized memory copy. */
 130 #if LJ_TARGET_X86
 131 #define CREC_COPY_REGWIN                2
 132 #elif LJ_TARGET_PPC || LJ_TARGET_MIPS
 133 #define CREC_COPY_REGWIN                8
 134 #else
 135 #define CREC_COPY_REGWIN                4
 136 #endif
 137 
 138 /* List of memory offsets for copy/fill. */
 139 typedef struct CRecMemList {
 140   CTSize ofs;           /* Offset in bytes. */
 141   IRType tp;            /* Type of load/store. */
 142   TRef trofs;           /* TRef of interned offset. */
 143   TRef trval;           /* TRef of load value. */
 144 } CRecMemList;
 145 
 146 /* Generate copy list for element-wise struct copy. */
 147 static MSize crec_copy_struct(CRecMemList *ml, CTState *cts, CType *ct)
 148 {
 149   CTypeID fid = ct->sib;
 150   MSize mlp = 0;
 151   while (fid) {
 152     CType *df = ctype_get(cts, fid);
 153     fid = df->sib;
 154     if (ctype_isfield(df->info)) {
 155       CType *cct;
 156       IRType tp;
 157       if (!gcref(df->name)) continue;  /* Ignore unnamed fields. */
 158       cct = ctype_rawchild(cts, df);  /* Field type. */
 159       tp = crec_ct2irt(cts, cct);
 160       if (tp == IRT_CDATA) return 0;  /* NYI: aggregates. */
 161       if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 162       ml[mlp].ofs = df->size;
 163       ml[mlp].tp = tp;
 164       mlp++;
 165       if (ctype_iscomplex(cct->info)) {
 166         if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 167         ml[mlp].ofs = df->size + (cct->size >> 1);
 168         ml[mlp].tp = tp;
 169         mlp++;
 170       }
 171     } else if (!ctype_isconstval(df->info)) {
 172       /* NYI: bitfields and sub-structures. */
 173       return 0;
 174     }
 175   }
 176   return mlp;
 177 }
 178 
 179 /* Generate unrolled copy list, from highest to lowest step size/alignment. */
 180 static MSize crec_copy_unroll(CRecMemList *ml, CTSize len, CTSize step,
 181                               IRType tp)
 182 {
 183   CTSize ofs = 0;
 184   MSize mlp = 0;
 185   if (tp == IRT_CDATA) tp = IRT_U8 + 2*lj_fls(step);
 186   do {
 187     while (ofs + step <= len) {
 188       if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 189       ml[mlp].ofs = ofs;
 190       ml[mlp].tp = tp;
 191       mlp++;
 192       ofs += step;
 193     }
 194     step >>= 1;
 195     tp -= 2;
 196   } while (ofs < len);
 197   return mlp;
 198 }
 199 
 200 /*
 201 ** Emit copy list with windowed loads/stores.
 202 ** LJ_TARGET_UNALIGNED: may emit unaligned loads/stores (not marked as such).
 203 */
 204 static void crec_copy_emit(jit_State *J, CRecMemList *ml, MSize mlp,
 205                            TRef trdst, TRef trsrc)
 206 {
 207   MSize i, j, rwin = 0;
 208   for (i = 0, j = 0; i < mlp; ) {
 209     TRef trofs = lj_ir_kintp(J, ml[i].ofs);
 210     TRef trsptr = emitir(IRT(IR_ADD, IRT_PTR), trsrc, trofs);
 211     ml[i].trval = emitir(IRT(IR_XLOAD, ml[i].tp), trsptr, 0);
 212     ml[i].trofs = trofs;
 213     i++;
 214     rwin += (LJ_SOFTFP && ml[i].tp == IRT_NUM) ? 2 : 1;
 215     if (rwin >= CREC_COPY_REGWIN || i >= mlp) {  /* Flush buffered stores. */
 216       rwin = 0;
 217       for ( ; j < i; j++) {
 218         TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, ml[j].trofs);
 219         emitir(IRT(IR_XSTORE, ml[j].tp), trdptr, ml[j].trval);
 220       }
 221     }
 222   }
 223 }
 224 
 225 /* Optimized memory copy. */
 226 static void crec_copy(jit_State *J, TRef trdst, TRef trsrc, TRef trlen,
 227                       CType *ct)
 228 {
 229   if (tref_isk(trlen)) {  /* Length must be constant. */
 230     CRecMemList ml[CREC_COPY_MAXUNROLL];
 231     MSize mlp = 0;
 232     CTSize step = 1, len = (CTSize)IR(tref_ref(trlen))->i;
 233     IRType tp = IRT_CDATA;
 234     int needxbar = 0;
 235     if (len == 0) return;  /* Shortcut. */
 236     if (len > CREC_COPY_MAXLEN) goto fallback;
 237     if (ct) {
 238       CTState *cts = ctype_ctsG(J2G(J));
 239       lua_assert(ctype_isarray(ct->info) || ctype_isstruct(ct->info));
 240       if (ctype_isarray(ct->info)) {
 241         CType *cct = ctype_rawchild(cts, ct);
 242         tp = crec_ct2irt(cts, cct);
 243         if (tp == IRT_CDATA) goto rawcopy;
 244         step = lj_ir_type_size[tp];
 245         lua_assert((len & (step-1)) == 0);
 246       } else if ((ct->info & CTF_UNION)) {
 247         step = (1u << ctype_align(ct->info));
 248         goto rawcopy;
 249       } else {
 250         mlp = crec_copy_struct(ml, cts, ct);
 251         goto emitcopy;
 252       }
 253     } else {
 254     rawcopy:
 255       needxbar = 1;
 256       if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
 257         step = CTSIZE_PTR;
 258     }
 259     mlp = crec_copy_unroll(ml, len, step, tp);
 260   emitcopy:
 261     if (mlp) {
 262       crec_copy_emit(J, ml, mlp, trdst, trsrc);
 263       if (needxbar)
 264         emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
 265       return;
 266     }
 267   }
 268 fallback:
 269   /* Call memcpy. Always needs a barrier to disable alias analysis. */
 270   lj_ir_call(J, IRCALL_memcpy, trdst, trsrc, trlen);
 271   emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
 272 }
 273 
 274 /* Generate unrolled fill list, from highest to lowest step size/alignment. */
 275 static MSize crec_fill_unroll(CRecMemList *ml, CTSize len, CTSize step)
 276 {
 277   CTSize ofs = 0;
 278   MSize mlp = 0;
 279   IRType tp = IRT_U8 + 2*lj_fls(step);
 280   do {
 281     while (ofs + step <= len) {
 282       if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 283       ml[mlp].ofs = ofs;
 284       ml[mlp].tp = tp;
 285       mlp++;
 286       ofs += step;
 287     }
 288     step >>= 1;
 289     tp -= 2;
 290   } while (ofs < len);
 291   return mlp;
 292 }
 293 
 294 /*
 295 ** Emit stores for fill list.
 296 ** LJ_TARGET_UNALIGNED: may emit unaligned stores (not marked as such).
 297 */
 298 static void crec_fill_emit(jit_State *J, CRecMemList *ml, MSize mlp,
 299                            TRef trdst, TRef trfill)
 300 {
 301   MSize i;
 302   for (i = 0; i < mlp; i++) {
 303     TRef trofs = lj_ir_kintp(J, ml[i].ofs);
 304     TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, trofs);
 305     emitir(IRT(IR_XSTORE, ml[i].tp), trdptr, trfill);
 306   }
 307 }
 308 
 309 /* Optimized memory fill. */
 310 static void crec_fill(jit_State *J, TRef trdst, TRef trlen, TRef trfill,
 311                       CTSize step)
 312 {
 313   if (tref_isk(trlen)) {  /* Length must be constant. */
 314     CRecMemList ml[CREC_FILL_MAXUNROLL];
 315     MSize mlp;
 316     CTSize len = (CTSize)IR(tref_ref(trlen))->i;
 317     if (len == 0) return;  /* Shortcut. */
 318     if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
 319       step = CTSIZE_PTR;
 320     if (step * CREC_FILL_MAXUNROLL < len) goto fallback;
 321     mlp = crec_fill_unroll(ml, len, step);
 322     if (!mlp) goto fallback;
 323     if (tref_isk(trfill) || ml[0].tp != IRT_U8)
 324       trfill = emitconv(trfill, IRT_INT, IRT_U8, 0);
 325     if (ml[0].tp != IRT_U8) {  /* Scatter U8 to U16/U32/U64. */
 326       if (CTSIZE_PTR == 8 && ml[0].tp == IRT_U64) {
 327         if (tref_isk(trfill))  /* Pointless on x64 with zero-extended regs. */
 328           trfill = emitconv(trfill, IRT_U64, IRT_U32, 0);
 329         trfill = emitir(IRT(IR_MUL, IRT_U64), trfill,
 330                         lj_ir_kint64(J, U64x(01010101,01010101)));
 331       } else {
 332         trfill = emitir(IRTI(IR_MUL), trfill,
 333                    lj_ir_kint(J, ml[0].tp == IRT_U16 ? 0x0101 : 0x01010101));
 334       }
 335     }
 336     crec_fill_emit(J, ml, mlp, trdst, trfill);
 337   } else {
 338 fallback:
 339     /* Call memset. Always needs a barrier to disable alias analysis. */
 340     lj_ir_call(J, IRCALL_memset, trdst, trfill, trlen);  /* Note: arg order! */
 341   }
 342   emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
 343 }
 344 
 345 /* -- Convert C type to C type -------------------------------------------- */
 346 
 347 /*
 348 ** This code mirrors the code in lj_cconv.c. It performs the same steps
 349 ** for the trace recorder that lj_cconv.c does for the interpreter.
 350 **
 351 ** One major difference is that we can get away with much fewer checks
 352 ** here. E.g. checks for casts, constness or correct types can often be
 353 ** omitted, even if they might fail. The interpreter subsequently throws
 354 ** an error, which aborts the trace.
 355 **
 356 ** All operations are specialized to their C types, so the on-trace
 357 ** outcome must be the same as the outcome in the interpreter. If the
 358 ** interpreter doesn't throw an error, then the trace is correct, too.
 359 ** Care must be taken not to generate invalid (temporary) IR or to
 360 ** trigger asserts.
 361 */
 362 
 363 /* Determine whether a passed number or cdata number is non-zero. */
 364 static int crec_isnonzero(CType *s, void *p)
 365 {
 366   if (p == (void *)0)
 367     return 0;
 368   if (p == (void *)1)
 369     return 1;
 370   if ((s->info & CTF_FP)) {
 371     if (s->size == sizeof(float))
 372       return (*(float *)p != 0);
 373     else
 374       return (*(double *)p != 0);
 375   } else {
 376     if (s->size == 1)
 377       return (*(uint8_t *)p != 0);
 378     else if (s->size == 2)
 379       return (*(uint16_t *)p != 0);
 380     else if (s->size == 4)
 381       return (*(uint32_t *)p != 0);
 382     else
 383       return (*(uint64_t *)p != 0);
 384   }
 385 }
 386 
 387 static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
 388                        void *svisnz)
 389 {
 390   IRType dt = crec_ct2irt(ctype_ctsG(J2G(J)), d);
 391   IRType st = crec_ct2irt(ctype_ctsG(J2G(J)), s);
 392   CTSize dsize = d->size, ssize = s->size;
 393   CTInfo dinfo = d->info, sinfo = s->info;
 394 
 395   if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT)
 396     goto err_conv;
 397 
 398   /*
 399   ** Note: Unlike lj_cconv_ct_ct(), sp holds the _value_ of pointers and
 400   ** numbers up to 8 bytes. Otherwise sp holds a pointer.
 401   */
 402 
 403   switch (cconv_idx2(dinfo, sinfo)) {
 404   /* Destination is a bool. */
 405   case CCX(B, B):
 406     goto xstore;  /* Source operand is already normalized. */
 407   case CCX(B, I):
 408   case CCX(B, F):
 409     if (st != IRT_CDATA) {
 410       /* Specialize to the result of a comparison against 0. */
 411       TRef zero = (st == IRT_NUM  || st == IRT_FLOAT) ? lj_ir_knum(J, 0) :
 412                   (st == IRT_I64 || st == IRT_U64) ? lj_ir_kint64(J, 0) :
 413                   lj_ir_kint(J, 0);
 414       int isnz = crec_isnonzero(s, svisnz);
 415       emitir(IRTG(isnz ? IR_NE : IR_EQ, st), sp, zero);
 416       sp = lj_ir_kint(J, isnz);
 417       goto xstore;
 418     }
 419     goto err_nyi;
 420 
 421   /* Destination is an integer. */
 422   case CCX(I, B):
 423   case CCX(I, I):
 424   conv_I_I:
 425     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 426     /* Extend 32 to 64 bit integer. */
 427     if (dsize == 8 && ssize < 8 && !(LJ_64 && (sinfo & CTF_UNSIGNED)))
 428       sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st,
 429                     (sinfo & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
 430     else if (dsize < 8 && ssize == 8)  /* Truncate from 64 bit integer. */
 431       sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, 0);
 432     else if (st == IRT_INT)
 433       sp = lj_opt_narrow_toint(J, sp);
 434   xstore:
 435     if (dt == IRT_I64 || dt == IRT_U64) lj_needsplit(J);
 436     if (dp == 0) return sp;
 437     emitir(IRT(IR_XSTORE, dt), dp, sp);
 438     break;
 439   case CCX(I, C):
 440     sp = emitir(IRT(IR_XLOAD, st), sp, 0);  /* Load re. */
 441     /* fallthrough */
 442   case CCX(I, F):
 443     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 444     sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_TRUNC|IRCONV_ANY);
 445     goto xstore;
 446   case CCX(I, P):
 447   case CCX(I, A):
 448     sinfo = CTINFO(CT_NUM, CTF_UNSIGNED);
 449     ssize = CTSIZE_PTR;
 450     st = IRT_UINTP;
 451     if (((dsize ^ ssize) & 8) == 0) {  /* Must insert no-op type conversion. */
 452       sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, IRT_PTR, 0);
 453       goto xstore;
 454     }
 455     goto conv_I_I;
 456 
 457   /* Destination is a floating-point number. */
 458   case CCX(F, B):
 459   case CCX(F, I):
 460   conv_F_I:
 461     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 462     sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st, 0);
 463     goto xstore;
 464   case CCX(F, C):
 465     sp = emitir(IRT(IR_XLOAD, st), sp, 0);  /* Load re. */
 466     /* fallthrough */
 467   case CCX(F, F):
 468   conv_F_F:
 469     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 470     if (dt != st) sp = emitconv(sp, dt, st, 0);
 471     goto xstore;
 472 
 473   /* Destination is a complex number. */
 474   case CCX(C, I):
 475   case CCX(C, F):
 476     {  /* Clear im. */
 477       TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
 478       emitir(IRT(IR_XSTORE, dt), ptr, lj_ir_knum(J, 0));
 479     }
 480     /* Convert to re. */
 481     if ((sinfo & CTF_FP)) goto conv_F_F; else goto conv_F_I;
 482 
 483   case CCX(C, C):
 484     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
 485     {
 486       TRef re, im, ptr;
 487       re = emitir(IRT(IR_XLOAD, st), sp, 0);
 488       ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, (ssize >> 1)));
 489       im = emitir(IRT(IR_XLOAD, st), ptr, 0);
 490       if (dt != st) {
 491         re = emitconv(re, dt, st, 0);
 492         im = emitconv(im, dt, st, 0);
 493       }
 494       emitir(IRT(IR_XSTORE, dt), dp, re);
 495       ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
 496       emitir(IRT(IR_XSTORE, dt), ptr, im);
 497     }
 498     break;
 499 
 500   /* Destination is a vector. */
 501   case CCX(V, I):
 502   case CCX(V, F):
 503   case CCX(V, C):
 504   case CCX(V, V):
 505     goto err_nyi;
 506 
 507   /* Destination is a pointer. */
 508   case CCX(P, P):
 509   case CCX(P, A):
 510   case CCX(P, S):
 511     /* There are only 32 bit pointers/addresses on 32 bit machines.
 512     ** Also ok on x64, since all 32 bit ops clear the upper part of the reg.
 513     */
 514     goto xstore;
 515   case CCX(P, I):
 516     if (st == IRT_CDATA) goto err_nyi;
 517     if (!LJ_64 && ssize == 8)  /* Truncate from 64 bit integer. */
 518       sp = emitconv(sp, IRT_U32, st, 0);
 519     goto xstore;
 520   case CCX(P, F):
 521     if (st == IRT_CDATA) goto err_nyi;
 522     /* The signed conversion is cheaper. x64 really has 47 bit pointers. */
 523     sp = emitconv(sp, (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32,
 524                   st, IRCONV_TRUNC|IRCONV_ANY);
 525     goto xstore;
 526 
 527   /* Destination is an array. */
 528   case CCX(A, A):
 529   /* Destination is a struct/union. */
 530   case CCX(S, S):
 531     if (dp == 0) goto err_conv;
 532     crec_copy(J, dp, sp, lj_ir_kint(J, dsize), d);
 533     break;
 534 
 535   default:
 536   err_conv:
 537   err_nyi:
 538     lj_trace_err(J, LJ_TRERR_NYICONV);
 539     break;
 540   }
 541   return 0;
 542 }
 543 
 544 /* -- Convert C type to TValue (load) ------------------------------------- */
 545 
 546 static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
 547 {
 548   CTState *cts = ctype_ctsG(J2G(J));
 549   IRType t = crec_ct2irt(cts, s);
 550   CTInfo sinfo = s->info;
 551   if (ctype_isnum(sinfo)) {
 552     TRef tr;
 553     if (t == IRT_CDATA)
 554       goto err_nyi;  /* NYI: copyval of >64 bit integers. */
 555     tr = emitir(IRT(IR_XLOAD, t), sp, 0);
 556     if (t == IRT_FLOAT || t == IRT_U32) {  /* Keep uint32_t/float as numbers. */
 557       return emitconv(tr, IRT_NUM, t, 0);
 558     } else if (t == IRT_I64 || t == IRT_U64) {  /* Box 64 bit integer. */
 559       sp = tr;
 560       lj_needsplit(J);
 561     } else if ((sinfo & CTF_BOOL)) {
 562       /* Assume not equal to zero. Fixup and emit pending guard later. */
 563       lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
 564       J->postproc = LJ_POST_FIXGUARD;
 565       return TREF_TRUE;
 566     } else {
 567       return tr;
 568     }
 569   } else if (ctype_isptr(sinfo) || ctype_isenum(sinfo)) {
 570     sp = emitir(IRT(IR_XLOAD, t), sp, 0);  /* Box pointers and enums. */
 571   } else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) {
 572     cts->L = J->L;
 573     sid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR);  /* Create ref. */
 574   } else if (ctype_iscomplex(sinfo)) {  /* Unbox/box complex. */
 575     ptrdiff_t esz = (ptrdiff_t)(s->size >> 1);
 576     TRef ptr, tr1, tr2, dp;
 577     dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL);
 578     tr1 = emitir(IRT(IR_XLOAD, t), sp, 0);
 579     ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, esz));
 580     tr2 = emitir(IRT(IR_XLOAD, t), ptr, 0);
 581     ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)));
 582     emitir(IRT(IR_XSTORE, t), ptr, tr1);
 583     ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)+esz));
 584     emitir(IRT(IR_XSTORE, t), ptr, tr2);
 585     return dp;
 586   } else {
 587     /* NYI: copyval of vectors. */
 588   err_nyi:
 589     lj_trace_err(J, LJ_TRERR_NYICONV);
 590   }
 591   /* Box pointer, ref, enum or 64 bit integer. */
 592   return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, sid), sp);
 593 }
 594 
 595 /* -- Convert TValue to C type (store) ------------------------------------ */
 596 
 597 static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
 598 {
 599   CTState *cts = ctype_ctsG(J2G(J));
 600   CTypeID sid = CTID_P_VOID;
 601   void *svisnz = 0;
 602   CType *s;
 603   if (LJ_LIKELY(tref_isinteger(sp))) {
 604     sid = CTID_INT32;
 605     svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
 606   } else if (tref_isnum(sp)) {
 607     sid = CTID_DOUBLE;
 608     svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
 609   } else if (tref_isbool(sp)) {
 610     sp = lj_ir_kint(J, tref_istrue(sp) ? 1 : 0);
 611     sid = CTID_BOOL;
 612   } else if (tref_isnil(sp)) {
 613     sp = lj_ir_kptr(J, NULL);
 614   } else if (tref_isudata(sp)) {
 615     GCudata *ud = udataV(sval);
 616     if (ud->udtype == UDTYPE_IO_FILE) {
 617       TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), sp, IRFL_UDATA_UDTYPE);
 618       emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, UDTYPE_IO_FILE));
 619       sp = emitir(IRT(IR_FLOAD, IRT_PTR), sp, IRFL_UDATA_FILE);
 620     } else {
 621       sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCudata)));
 622     }
 623   } else if (tref_isstr(sp)) {
 624     if (ctype_isenum(d->info)) {  /* Match string against enum constant. */
 625       GCstr *str = strV(sval);
 626       CTSize ofs;
 627       CType *cct = lj_ctype_getfield(cts, d, str, &ofs);
 628       /* Specialize to the name of the enum constant. */
 629       emitir(IRTG(IR_EQ, IRT_STR), sp, lj_ir_kstr(J, str));
 630       if (cct && ctype_isconstval(cct->info)) {
 631         lua_assert(ctype_child(cts, cct)->size == 4);
 632         svisnz = (void *)(intptr_t)(ofs != 0);
 633         sp = lj_ir_kint(J, (int32_t)ofs);
 634         sid = ctype_cid(cct->info);
 635       }  /* else: interpreter will throw. */
 636     } else if (ctype_isrefarray(d->info)) {  /* Copy string to array. */
 637       lj_trace_err(J, LJ_TRERR_BADTYPE);  /* NYI */
 638     } else {  /* Otherwise pass the string data as a const char[]. */
 639       /* Don't use STRREF. It folds with SNEW, which loses the trailing NUL. */
 640       sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCstr)));
 641       sid = CTID_A_CCHAR;
 642     }
 643   } else {  /* NYI: tref_istab(sp), tref_islightud(sp). */
 644     IRType t;
 645     sid = argv2cdata(J, sp, sval)->ctypeid;
 646     s = ctype_raw(cts, sid);
 647     svisnz = cdataptr(cdataV(sval));
 648     t = crec_ct2irt(cts, s);
 649     if (ctype_isptr(s->info)) {
 650       sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_PTR);
 651       if (ctype_isref(s->info)) {
 652         svisnz = *(void **)svisnz;
 653         s = ctype_rawchild(cts, s);
 654         if (ctype_isenum(s->info)) s = ctype_child(cts, s);
 655         t = crec_ct2irt(cts, s);
 656       } else {
 657         goto doconv;
 658       }
 659     } else if (t == IRT_I64 || t == IRT_U64) {
 660       sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT64);
 661       lj_needsplit(J);
 662       goto doconv;
 663     } else if (t == IRT_INT || t == IRT_U32) {
 664       if (ctype_isenum(s->info)) s = ctype_child(cts, s);
 665       sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT);
 666       goto doconv;
 667     } else {
 668       sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCcdata)));
 669     }
 670     if (ctype_isnum(s->info) && t != IRT_CDATA)
 671       sp = emitir(IRT(IR_XLOAD, t), sp, 0);  /* Load number value. */
 672     goto doconv;
 673   }
 674   s = ctype_get(cts, sid);
 675 doconv:
 676   if (ctype_isenum(d->info)) d = ctype_child(cts, d);
 677   return crec_ct_ct(J, d, s, dp, sp, svisnz);
 678 }
 679 
 680 /* -- C data metamethods -------------------------------------------------- */
 681 
 682 /* This would be rather difficult in FOLD, so do it here:
 683 ** (base+k)+(idx*sz)+ofs ==> (base+idx*sz)+(ofs+k)
 684 ** (base+(idx+k)*sz)+ofs ==> (base+idx*sz)+(ofs+k*sz)
 685 */
 686 static TRef crec_reassoc_ofs(jit_State *J, TRef tr, ptrdiff_t *ofsp, MSize sz)
 687 {
 688   IRIns *ir = IR(tref_ref(tr));
 689   if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD) && irref_isk(ir->op2) &&
 690       (ir->o == IR_ADD || ir->o == IR_ADDOV || ir->o == IR_SUBOV)) {
 691     IRIns *irk = IR(ir->op2);
 692     ptrdiff_t k;
 693     if (LJ_64 && irk->o == IR_KINT64)
 694       k = (ptrdiff_t)ir_kint64(irk)->u64 * sz;
 695     else
 696       k = (ptrdiff_t)irk->i * sz;
 697     if (ir->o == IR_SUBOV) *ofsp -= k; else *ofsp += k;
 698     tr = ir->op1;  /* Not a TRef, but the caller doesn't care. */
 699   }
 700   return tr;
 701 }
 702 
 703 /* Record ctype __index/__newindex metamethods. */
 704 static void crec_index_meta(jit_State *J, CTState *cts, CType *ct,
 705                             RecordFFData *rd)
 706 {
 707   CTypeID id = ctype_typeid(cts, ct);
 708   cTValue *tv = lj_ctype_meta(cts, id, rd->data ? MM_newindex : MM_index);
 709   if (!tv)
 710     lj_trace_err(J, LJ_TRERR_BADTYPE);
 711   if (tvisfunc(tv)) {
 712     J->base[-1] = lj_ir_kfunc(J, funcV(tv)) | TREF_FRAME;
 713     rd->nres = -1;  /* Pending tailcall. */
 714   } else if (rd->data == 0 && tvistab(tv) && tref_isstr(J->base[1])) {
 715     /* Specialize to result of __index lookup. */
 716     cTValue *o = lj_tab_get(J->L, tabV(tv), &rd->argv[1]);
 717     J->base[0] = lj_record_constify(J, o);
 718     if (!J->base[0])
 719       lj_trace_err(J, LJ_TRERR_BADTYPE);
 720     /* Always specialize to the key. */
 721     emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
 722   } else {
 723     /* NYI: resolving of non-function metamethods. */
 724     /* NYI: non-string keys for __index table. */
 725     /* NYI: stores to __newindex table. */
 726     lj_trace_err(J, LJ_TRERR_BADTYPE);
 727   }
 728 }
 729 
 730 void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd)
 731 {
 732   TRef idx, ptr = J->base[0];
 733   ptrdiff_t ofs = sizeof(GCcdata);
 734   GCcdata *cd = argv2cdata(J, ptr, &rd->argv[0]);
 735   CTState *cts = ctype_ctsG(J2G(J));
 736   CType *ct = ctype_raw(cts, cd->ctypeid);
 737   CTypeID sid = 0;
 738 
 739   /* Resolve pointer or reference for cdata object. */
 740   if (ctype_isptr(ct->info)) {
 741     IRType t = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
 742     if (ctype_isref(ct->info)) ct = ctype_rawchild(cts, ct);
 743     ptr = emitir(IRT(IR_FLOAD, t), ptr, IRFL_CDATA_PTR);
 744     ofs = 0;
 745     ptr = crec_reassoc_ofs(J, ptr, &ofs, 1);
 746   }
 747 
 748 again:
 749   idx = J->base[1];
 750   if (tref_isnumber(idx)) {
 751     idx = lj_opt_narrow_cindex(J, idx);
 752     if (ctype_ispointer(ct->info)) {
 753       CTSize sz;
 754   integer_key:
 755       if ((ct->info & CTF_COMPLEX))
 756         idx = emitir(IRT(IR_BAND, IRT_INTP), idx, lj_ir_kintp(J, 1));
 757       sz = lj_ctype_size(cts, (sid = ctype_cid(ct->info)));
 758       idx = crec_reassoc_ofs(J, idx, &ofs, sz);
 759 #if LJ_TARGET_ARM || LJ_TARGET_PPC
 760       /* Hoist base add to allow fusion of index/shift into operands. */
 761       if (LJ_LIKELY(J->flags & JIT_F_OPT_LOOP) && ofs
 762 #if LJ_TARGET_ARM
 763           && (sz == 1 || sz == 4)
 764 #endif
 765           ) {
 766         ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
 767         ofs = 0;
 768       }
 769 #endif
 770       idx = emitir(IRT(IR_MUL, IRT_INTP), idx, lj_ir_kintp(J, sz));
 771       ptr = emitir(IRT(IR_ADD, IRT_PTR), idx, ptr);
 772     }
 773   } else if (tref_iscdata(idx)) {
 774     GCcdata *cdk = cdataV(&rd->argv[1]);
 775     CType *ctk = ctype_raw(cts, cdk->ctypeid);
 776     IRType t = crec_ct2irt(cts, ctk);
 777     if (ctype_ispointer(ct->info) && t >= IRT_I8 && t <= IRT_U64) {
 778       if (ctk->size == 8) {
 779         idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT64);
 780       } else if (ctk->size == 4) {
 781         idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT);
 782       } else {
 783         idx = emitir(IRT(IR_ADD, IRT_PTR), idx,
 784                      lj_ir_kintp(J, sizeof(GCcdata)));
 785         idx = emitir(IRT(IR_XLOAD, t), idx, 0);
 786       }
 787       if (LJ_64 && ctk->size < sizeof(intptr_t) && !(ctk->info & CTF_UNSIGNED))
 788         idx = emitconv(idx, IRT_INTP, IRT_INT, IRCONV_SEXT);
 789       if (!LJ_64 && ctk->size > sizeof(intptr_t)) {
 790         idx = emitconv(idx, IRT_INTP, t, 0);
 791         lj_needsplit(J);
 792       }
 793       goto integer_key;
 794     }
 795   } else if (tref_isstr(idx)) {
 796     GCstr *name = strV(&rd->argv[1]);
 797     if (cd && cd->ctypeid == CTID_CTYPEID)
 798       ct = ctype_raw(cts, crec_constructor(J, cd, ptr));
 799     if (ctype_isstruct(ct->info)) {
 800       CTSize fofs;
 801       CType *fct;
 802       fct = lj_ctype_getfield(cts, ct, name, &fofs);
 803       if (fct) {
 804         /* Always specialize to the field name. */
 805         emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
 806         if (ctype_isconstval(fct->info)) {
 807           if (fct->size >= 0x80000000u &&
 808               (ctype_child(cts, fct)->info & CTF_UNSIGNED)) {
 809             J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)fct->size);
 810             return;
 811           }
 812           J->base[0] = lj_ir_kint(J, (int32_t)fct->size);
 813           return;  /* Interpreter will throw for newindex. */
 814         } else if (ctype_isbitfield(fct->info)) {
 815           lj_trace_err(J, LJ_TRERR_NYICONV);
 816         } else {
 817           lua_assert(ctype_isfield(fct->info));
 818           sid = ctype_cid(fct->info);
 819         }
 820         ofs += (ptrdiff_t)fofs;
 821       }
 822     } else if (ctype_iscomplex(ct->info)) {
 823       if (name->len == 2 &&
 824           ((strdata(name)[0] == 'r' && strdata(name)[1] == 'e') ||
 825            (strdata(name)[0] == 'i' && strdata(name)[1] == 'm'))) {
 826         /* Always specialize to the field name. */
 827         emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
 828         if (strdata(name)[0] == 'i') ofs += (ct->size >> 1);
 829         sid = ctype_cid(ct->info);
 830       }
 831     }
 832   }
 833   if (!sid) {
 834     if (ctype_isptr(ct->info)) {  /* Automatically perform '->'. */
 835       CType *cct = ctype_rawchild(cts, ct);
 836       if (ctype_isstruct(cct->info)) {
 837         ct = cct;
 838         cd = NULL;
 839         if (tref_isstr(idx)) goto again;
 840       }
 841     }
 842     crec_index_meta(J, cts, ct, rd);
 843     return;
 844   }
 845 
 846   if (ofs)
 847     ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
 848 
 849   /* Resolve reference for field. */
 850   ct = ctype_get(cts, sid);
 851   if (ctype_isref(ct->info)) {
 852     ptr = emitir(IRT(IR_XLOAD, IRT_PTR), ptr, 0);
 853     sid = ctype_cid(ct->info);
 854     ct = ctype_get(cts, sid);
 855   }
 856 
 857   while (ctype_isattrib(ct->info))
 858     ct = ctype_child(cts, ct);  /* Skip attributes. */
 859 
 860   if (rd->data == 0) {  /* __index metamethod. */
 861     J->base[0] = crec_tv_ct(J, ct, sid, ptr);
 862   } else {  /* __newindex metamethod. */
 863     rd->nres = 0;
 864     J->needsnap = 1;
 865     crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
 866   }
 867 }
 868 
 869 /* Record setting a finalizer. */
 870 static void crec_finalizer(jit_State *J, TRef trcd, cTValue *fin)
 871 {
 872   TRef trlo = lj_ir_call(J, IRCALL_lj_cdata_setfin, trcd);
 873   TRef trhi = emitir(IRT(IR_ADD, IRT_P32), trlo, lj_ir_kint(J, 4));
 874   if (LJ_BE) { TRef tmp = trlo; trlo = trhi; trhi = tmp; }
 875   if (tvisfunc(fin)) {
 876     emitir(IRT(IR_XSTORE, IRT_P32), trlo, lj_ir_kfunc(J, funcV(fin)));
 877     emitir(IRTI(IR_XSTORE), trhi, lj_ir_kint(J, LJ_TFUNC));
 878   } else if (tviscdata(fin)) {
 879     emitir(IRT(IR_XSTORE, IRT_P32), trlo,
 880            lj_ir_kgc(J, obj2gco(cdataV(fin)), IRT_CDATA));
 881     emitir(IRTI(IR_XSTORE), trhi, lj_ir_kint(J, LJ_TCDATA));
 882   } else {
 883     lj_trace_err(J, LJ_TRERR_BADTYPE);
 884   }
 885   J->needsnap = 1;
 886 }
 887 
 888 /* Record cdata allocation. */
 889 static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
 890 {
 891   CTState *cts = ctype_ctsG(J2G(J));
 892   CTSize sz;
 893   CTInfo info = lj_ctype_info(cts, id, &sz);
 894   CType *d = ctype_raw(cts, id);
 895   TRef trid;
 896   if (!sz || sz > 128 || (info & CTF_VLA) || ctype_align(info) > CT_MEMALIGN)
 897     lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: large/special allocations. */
 898   trid = lj_ir_kint(J, id);
 899   /* Use special instruction to box pointer or 32/64 bit integer. */
 900   if (ctype_isptr(info) || (ctype_isinteger(info) && (sz == 4 || sz == 8))) {
 901     TRef sp = J->base[1] ? crec_ct_tv(J, d, 0, J->base[1], &rd->argv[1]) :
 902               ctype_isptr(info) ? lj_ir_kptr(J, NULL) :
 903               sz == 4 ? lj_ir_kint(J, 0) :
 904               (lj_needsplit(J), lj_ir_kint64(J, 0));
 905     J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, sp);
 906   } else {
 907     TRef trcd = emitir(IRTG(IR_CNEW, IRT_CDATA), trid, TREF_NIL);
 908     cTValue *fin;
 909     J->base[0] = trcd;
 910     if (J->base[1] && !J->base[2] &&
 911         !lj_cconv_multi_init(cts, d, &rd->argv[1])) {
 912       goto single_init;
 913     } else if (ctype_isarray(d->info)) {
 914       CType *dc = ctype_rawchild(cts, d);  /* Array element type. */
 915       CTSize ofs, esize = dc->size;
 916       TRef sp = 0;
 917       TValue tv;
 918       TValue *sval = &tv;
 919       MSize i;
 920       tv.u64 = 0;
 921       if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info)))
 922         lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init array of aggregates. */
 923       for (i = 1, ofs = 0; ofs < sz; ofs += esize) {
 924         TRef dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
 925                          lj_ir_kintp(J, ofs + sizeof(GCcdata)));
 926         if (J->base[i]) {
 927           sp = J->base[i];
 928           sval = &rd->argv[i];
 929           i++;
 930         } else if (i != 2) {
 931           sp = ctype_isnum(dc->info) ? lj_ir_kint(J, 0) : TREF_NIL;
 932         }
 933         crec_ct_tv(J, dc, dp, sp, sval);
 934       }
 935     } else if (ctype_isstruct(d->info)) {
 936       CTypeID fid = d->sib;
 937       MSize i = 1;
 938       while (fid) {
 939         CType *df = ctype_get(cts, fid);
 940         fid = df->sib;
 941         if (ctype_isfield(df->info)) {
 942           CType *dc;
 943           TRef sp, dp;
 944           TValue tv;
 945           TValue *sval = &tv;
 946           setintV(&tv, 0);
 947           if (!gcref(df->name)) continue;  /* Ignore unnamed fields. */
 948           dc = ctype_rawchild(cts, df);  /* Field type. */
 949           if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info) ||
 950                 ctype_isenum(dc->info)))
 951             lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init aggregates. */
 952           if (J->base[i]) {
 953             sp = J->base[i];
 954             sval = &rd->argv[i];
 955             i++;
 956           } else {
 957             sp = ctype_isptr(dc->info) ? TREF_NIL : lj_ir_kint(J, 0);
 958           }
 959           dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
 960                       lj_ir_kintp(J, df->size + sizeof(GCcdata)));
 961           crec_ct_tv(J, dc, dp, sp, sval);
 962         } else if (!ctype_isconstval(df->info)) {
 963           /* NYI: init bitfields and sub-structures. */
 964           lj_trace_err(J, LJ_TRERR_NYICONV);
 965         }
 966       }
 967     } else {
 968       TRef dp;
 969     single_init:
 970       dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
 971       if (J->base[1]) {
 972         crec_ct_tv(J, d, dp, J->base[1], &rd->argv[1]);
 973       } else {
 974         TValue tv;
 975         tv.u64 = 0;
 976         crec_ct_tv(J, d, dp, lj_ir_kint(J, 0), &tv);
 977       }
 978     }
 979     /* Handle __gc metamethod. */
 980     fin = lj_ctype_meta(cts, id, MM_gc);
 981     if (fin)
 982       crec_finalizer(J, trcd, fin);
 983   }
 984 }
 985 
 986 /* Record argument conversions. */
 987 static TRef crec_call_args(jit_State *J, RecordFFData *rd,
 988                            CTState *cts, CType *ct)
 989 {
 990   TRef args[CCI_NARGS_MAX];
 991   CTypeID fid;
 992   MSize i, n;
 993   TRef tr, *base;
 994   cTValue *o;
 995 #if LJ_TARGET_X86
 996 #if LJ_ABI_WIN
 997   TRef *arg0 = NULL, *arg1 = NULL;
 998 #endif
 999   int ngpr = 0;
1000   if (ctype_cconv(ct->info) == CTCC_THISCALL)
1001     ngpr = 1;
1002   else if (ctype_cconv(ct->info) == CTCC_FASTCALL)
1003     ngpr = 2;
1004 #endif
1005 
1006   /* Skip initial attributes. */
1007   fid = ct->sib;
1008   while (fid) {
1009     CType *ctf = ctype_get(cts, fid);
1010     if (!ctype_isattrib(ctf->info)) break;
1011     fid = ctf->sib;
1012   }
1013   args[0] = TREF_NIL;
1014   for (n = 0, base = J->base+1, o = rd->argv+1; *base; n++, base++, o++) {
1015     CTypeID did;
1016     CType *d;
1017 
1018     if (n >= CCI_NARGS_MAX)
1019       lj_trace_err(J, LJ_TRERR_NYICALL);
1020 
1021     if (fid) {  /* Get argument type from field. */
1022       CType *ctf = ctype_get(cts, fid);
1023       fid = ctf->sib;
1024       lua_assert(ctype_isfield(ctf->info));
1025       did = ctype_cid(ctf->info);
1026     } else {
1027       if (!(ct->info & CTF_VARARG))
1028         lj_trace_err(J, LJ_TRERR_NYICALL);  /* Too many arguments. */
1029       did = lj_ccall_ctid_vararg(cts, o);  /* Infer vararg type. */
1030     }
1031     d = ctype_raw(cts, did);
1032     if (!(ctype_isnum(d->info) || ctype_isptr(d->info) ||
1033           ctype_isenum(d->info)))
1034       lj_trace_err(J, LJ_TRERR_NYICALL);
1035     tr = crec_ct_tv(J, d, 0, *base, o);
1036     if (ctype_isinteger_or_bool(d->info)) {
1037       if (d->size < 4) {
1038         if ((d->info & CTF_UNSIGNED))
1039           tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_U8 : IRT_U16, 0);
1040         else
1041           tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_I8 : IRT_I16,IRCONV_SEXT);
1042       }
1043     } else if (LJ_SOFTFP && ctype_isfp(d->info) && d->size > 4) {
1044       lj_needsplit(J);
1045     }
1046 #if LJ_TARGET_X86
1047     /* 64 bit args must not end up in registers for fastcall/thiscall. */
1048 #if LJ_ABI_WIN
1049     if (!ctype_isfp(d->info)) {
1050       /* Sigh, the Windows/x86 ABI allows reordering across 64 bit args. */
1051       if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1052         if (ngpr) {
1053           arg0 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1054           if (ngpr) {
1055             arg1 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1056           }
1057         }
1058       } else {
1059         if (arg0) { *arg0 = tr; arg0 = NULL; n--; continue; }
1060         if (arg1) { *arg1 = tr; arg1 = NULL; n--; continue; }
1061         if (ngpr) ngpr--;
1062       }
1063     }
1064 #else
1065     if (!ctype_isfp(d->info) && ngpr) {
1066       if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1067         /* No reordering for other x86 ABIs. Simply add alignment args. */
1068         do { args[n++] = TREF_NIL; } while (--ngpr);
1069       } else {
1070         ngpr--;
1071       }
1072     }
1073 #endif
1074 #endif
1075     args[n] = tr;
1076   }
1077   tr = args[0];
1078   for (i = 1; i < n; i++)
1079     tr = emitir(IRT(IR_CARG, IRT_NIL), tr, args[i]);
1080   return tr;
1081 }
1082 
1083 /* Create a snapshot for the caller, simulating a 'false' return value. */
1084 static void crec_snap_caller(jit_State *J)
1085 {
1086   lua_State *L = J->L;
1087   TValue *base = L->base, *top = L->top;
1088   const BCIns *pc = J->pc;
1089   TRef ftr = J->base[-1];
1090   ptrdiff_t delta;
1091   if (!frame_islua(base-1) || J->framedepth <= 0)
1092     lj_trace_err(J, LJ_TRERR_NYICALL);
1093   J->pc = frame_pc(base-1); delta = 1+bc_a(J->pc[-1]);
1094   L->top = base; L->base = base - delta;
1095   J->base[-1] = TREF_FALSE;
1096   J->base -= delta; J->baseslot -= (BCReg)delta;
1097   J->maxslot = (BCReg)delta; J->framedepth--;
1098   lj_snap_add(J);
1099   L->base = base; L->top = top;
1100   J->framedepth++; J->maxslot = 1;
1101   J->base += delta; J->baseslot += (BCReg)delta;
1102   J->base[-1] = ftr; J->pc = pc;
1103 }
1104 
1105 /* Record function call. */
1106 static int crec_call(jit_State *J, RecordFFData *rd, GCcdata *cd)
1107 {
1108   CTState *cts = ctype_ctsG(J2G(J));
1109   CType *ct = ctype_raw(cts, cd->ctypeid);
1110   IRType tp = IRT_PTR;
1111   if (ctype_isptr(ct->info)) {
1112     tp = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
1113     ct = ctype_rawchild(cts, ct);
1114   }
1115   if (ctype_isfunc(ct->info)) {
1116     TRef func = emitir(IRT(IR_FLOAD, tp), J->base[0], IRFL_CDATA_PTR);
1117     CType *ctr = ctype_rawchild(cts, ct);
1118     IRType t = crec_ct2irt(cts, ctr);
1119     TRef tr;
1120     TValue tv;
1121     /* Check for blacklisted C functions that might call a callback. */
1122     setlightudV(&tv,
1123                 cdata_getptr(cdataptr(cd), (LJ_64 && tp == IRT_P64) ? 8 : 4));
1124     if (tvistrue(lj_tab_get(J->L, cts->miscmap, &tv)))
1125       lj_trace_err(J, LJ_TRERR_BLACKL);
1126     if (ctype_isvoid(ctr->info)) {
1127       t = IRT_NIL;
1128       rd->nres = 0;
1129     } else if (!(ctype_isnum(ctr->info) || ctype_isptr(ctr->info) ||
1130                  ctype_isenum(ctr->info)) || t == IRT_CDATA) {
1131       lj_trace_err(J, LJ_TRERR_NYICALL);
1132     }
1133     if ((ct->info & CTF_VARARG)
1134 #if LJ_TARGET_X86
1135         || ctype_cconv(ct->info) != CTCC_CDECL
1136 #endif
1137         )
1138       func = emitir(IRT(IR_CARG, IRT_NIL), func,
1139                     lj_ir_kint(J, ctype_typeid(cts, ct)));
1140     tr = emitir(IRT(IR_CALLXS, t), crec_call_args(J, rd, cts, ct), func);
1141     if (ctype_isbool(ctr->info)) {
1142       if (frame_islua(J->L->base-1) && bc_b(frame_pc(J->L->base-1)[-1]) == 1) {
1143         /* Don't check result if ignored. */
1144         tr = TREF_NIL;
1145       } else {
1146         crec_snap_caller(J);
1147 #if LJ_TARGET_X86ORX64
1148         /* Note: only the x86/x64 backend supports U8 and only for EQ(tr, 0). */
1149         lj_ir_set(J, IRTG(IR_NE, IRT_U8), tr, lj_ir_kint(J, 0));
1150 #else
1151         lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
1152 #endif
1153         J->postproc = LJ_POST_FIXGUARDSNAP;
1154         tr = TREF_TRUE;
1155       }
1156     } else if (t == IRT_PTR || (LJ_64 && t == IRT_P32) ||
1157                t == IRT_I64 || t == IRT_U64 || ctype_isenum(ctr->info)) {
1158       TRef trid = lj_ir_kint(J, ctype_cid(ct->info));
1159       tr = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, tr);
1160       if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1161     } else if (t == IRT_FLOAT || t == IRT_U32) {
1162       tr = emitconv(tr, IRT_NUM, t, 0);
1163     } else if (t == IRT_I8 || t == IRT_I16) {
1164       tr = emitconv(tr, IRT_INT, t, IRCONV_SEXT);
1165     } else if (t == IRT_U8 || t == IRT_U16) {
1166       tr = emitconv(tr, IRT_INT, t, 0);
1167     }
1168     J->base[0] = tr;
1169     J->needsnap = 1;
1170     return 1;
1171   }
1172   return 0;
1173 }
1174 
1175 void LJ_FASTCALL recff_cdata_call(jit_State *J, RecordFFData *rd)
1176 {
1177   CTState *cts = ctype_ctsG(J2G(J));
1178   GCcdata *cd = argv2cdata(J, J->base[0], &rd->argv[0]);
1179   CTypeID id = cd->ctypeid;
1180   CType *ct;
1181   cTValue *tv;
1182   MMS mm = MM_call;
1183   if (id == CTID_CTYPEID) {
1184     id = crec_constructor(J, cd, J->base[0]);
1185     mm = MM_new;
1186   } else if (crec_call(J, rd, cd)) {
1187     return;
1188   }
1189   /* Record ctype __call/__new metamethod. */
1190   ct = ctype_raw(cts, id);
1191   tv = lj_ctype_meta(cts, ctype_isptr(ct->info) ? ctype_cid(ct->info) : id, mm);
1192   if (tv) {
1193     if (tvisfunc(tv)) {
1194       J->base[-1] = lj_ir_kfunc(J, funcV(tv)) | TREF_FRAME;
1195       rd->nres = -1;  /* Pending tailcall. */
1196       return;
1197     }
1198   } else if (mm == MM_new) {
1199     crec_alloc(J, rd, id);
1200     return;
1201   }
1202   /* No metamethod or NYI: non-function metamethods. */
1203   lj_trace_err(J, LJ_TRERR_BADTYPE);
1204 }
1205 
1206 static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
1207 {
1208   if (sp[0] && sp[1] && ctype_isnum(s[0]->info) && ctype_isnum(s[1]->info)) {
1209     IRType dt;
1210     CTypeID id;
1211     TRef tr;
1212     MSize i;
1213     IROp op;
1214     lj_needsplit(J);
1215     if (((s[0]->info & CTF_UNSIGNED) && s[0]->size == 8) ||
1216         ((s[1]->info & CTF_UNSIGNED) && s[1]->size == 8)) {
1217       dt = IRT_U64; id = CTID_UINT64;
1218     } else {
1219       dt = IRT_I64; id = CTID_INT64;
1220       if (mm < MM_add &&
1221           !((s[0]->info | s[1]->info) & CTF_FP) &&
1222           s[0]->size == 4 && s[1]->size == 4) {  /* Try to narrow comparison. */
1223         if (!((s[0]->info ^ s[1]->info) & CTF_UNSIGNED) ||
1224             (tref_isk(sp[1]) && IR(tref_ref(sp[1]))->i >= 0)) {
1225           dt = (s[0]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1226           goto comp;
1227         } else if (tref_isk(sp[0]) && IR(tref_ref(sp[0]))->i >= 0) {
1228           dt = (s[1]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1229           goto comp;
1230         }
1231       }
1232     }
1233     for (i = 0; i < 2; i++) {
1234       IRType st = tref_type(sp[i]);
1235       if (st == IRT_NUM || st == IRT_FLOAT)
1236         sp[i] = emitconv(sp[i], dt, st, IRCONV_TRUNC|IRCONV_ANY);
1237       else if (!(st == IRT_I64 || st == IRT_U64))
1238         sp[i] = emitconv(sp[i], dt, IRT_INT,
1239                          (s[i]->info & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
1240     }
1241     if (mm < MM_add) {
1242     comp:
1243       /* Assume true comparison. Fixup and emit pending guard later. */
1244       if (mm == MM_eq) {
1245         op = IR_EQ;
1246       } else {
1247         op = mm == MM_lt ? IR_LT : IR_LE;
1248         if (dt == IRT_U32 || dt == IRT_U64)
1249           op += (IR_ULT-IR_LT);
1250       }
1251       lj_ir_set(J, IRTG(op, dt), sp[0], sp[1]);
1252       J->postproc = LJ_POST_FIXGUARD;
1253       return TREF_TRUE;
1254     } else {
1255       tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, dt), sp[0], sp[1]);
1256     }
1257     return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1258   }
1259   return 0;
1260 }
1261 
1262 static TRef crec_arith_ptr(jit_State *J, TRef *sp, CType **s, MMS mm)
1263 {
1264   CTState *cts = ctype_ctsG(J2G(J));
1265   CType *ctp = s[0];
1266   if (!(sp[0] && sp[1])) return 0;
1267   if (ctype_isptr(ctp->info) || ctype_isrefarray(ctp->info)) {
1268     if ((mm == MM_sub || mm == MM_eq || mm == MM_lt || mm == MM_le) &&
1269         (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1270       if (mm == MM_sub) {  /* Pointer difference. */
1271         TRef tr;
1272         CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1273         if (sz == 0 || (sz & (sz-1)) != 0)
1274           return 0;  /* NYI: integer division. */
1275         tr = emitir(IRT(IR_SUB, IRT_INTP), sp[0], sp[1]);
1276         tr = emitir(IRT(IR_BSAR, IRT_INTP), tr, lj_ir_kint(J, lj_fls(sz)));
1277 #if LJ_64
1278         tr = emitconv(tr, IRT_NUM, IRT_INTP, 0);
1279 #endif
1280         return tr;
1281       } else {  /* Pointer comparison (unsigned). */
1282         /* Assume true comparison. Fixup and emit pending guard later. */
1283         IROp op = mm == MM_eq ? IR_EQ : mm == MM_lt ? IR_ULT : IR_ULE;
1284         lj_ir_set(J, IRTG(op, IRT_PTR), sp[0], sp[1]);
1285         J->postproc = LJ_POST_FIXGUARD;
1286         return TREF_TRUE;
1287       }
1288     }
1289     if (!((mm == MM_add || mm == MM_sub) && ctype_isnum(s[1]->info)))
1290       return 0;
1291   } else if (mm == MM_add && ctype_isnum(ctp->info) &&
1292              (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1293     TRef tr = sp[0]; sp[0] = sp[1]; sp[1] = tr;  /* Swap pointer and index. */
1294     ctp = s[1];
1295   } else {
1296     return 0;
1297   }
1298   {
1299     TRef tr = sp[1];
1300     IRType t = tref_type(tr);
1301     CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1302     CTypeID id;
1303 #if LJ_64
1304     if (t == IRT_NUM || t == IRT_FLOAT)
1305       tr = emitconv(tr, IRT_INTP, t, IRCONV_TRUNC|IRCONV_ANY);
1306     else if (!(t == IRT_I64 || t == IRT_U64))
1307       tr = emitconv(tr, IRT_INTP, IRT_INT,
1308                     ((t - IRT_I8) & 1) ? 0 : IRCONV_SEXT);
1309 #else
1310     if (!tref_typerange(sp[1], IRT_I8, IRT_U32)) {
1311       tr = emitconv(tr, IRT_INTP, t,
1312                     (t == IRT_NUM || t == IRT_FLOAT) ?
1313                     IRCONV_TRUNC|IRCONV_ANY : 0);
1314     }
1315 #endif
1316     tr = emitir(IRT(IR_MUL, IRT_INTP), tr, lj_ir_kintp(J, sz));
1317     tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, IRT_PTR), sp[0], tr);
1318     id = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(ctp->info)),
1319                          CTSIZE_PTR);
1320     return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1321   }
1322 }
1323 
1324 /* Record ctype arithmetic metamethods. */
1325 static TRef crec_arith_meta(jit_State *J, TRef *sp, CType **s, CTState *cts,
1326                             RecordFFData *rd)
1327 {
1328   cTValue *tv = NULL;
1329   if (J->base[0]) {
1330     if (tviscdata(&rd->argv[0])) {
1331       CTypeID id = argv2cdata(J, J->base[0], &rd->argv[0])->ctypeid;
1332       CType *ct = ctype_raw(cts, id);
1333       if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1334       tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1335     }
1336     if (!tv && J->base[1] && tviscdata(&rd->argv[1])) {
1337       CTypeID id = argv2cdata(J, J->base[1], &rd->argv[1])->ctypeid;
1338       CType *ct = ctype_raw(cts, id);
1339       if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1340       tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1341     }
1342   }
1343   if (tv) {
1344     if (tvisfunc(tv)) {
1345       J->base[-1] = lj_ir_kfunc(J, funcV(tv)) | TREF_FRAME;
1346       rd->nres = -1;  /* Pending tailcall. */
1347       return 0;
1348     }  /* NYI: non-function metamethods. */
1349   } else if ((MMS)rd->data == MM_eq) {  /* Fallback cdata pointer comparison. */
1350     if (sp[0] && sp[1] && ctype_isnum(s[0]->info) == ctype_isnum(s[1]->info)) {
1351       /* Assume true comparison. Fixup and emit pending guard later. */
1352       lj_ir_set(J, IRTG(IR_EQ, IRT_PTR), sp[0], sp[1]);
1353       J->postproc = LJ_POST_FIXGUARD;
1354       return TREF_TRUE;
1355     } else {
1356       return TREF_FALSE;
1357     }
1358   }
1359   lj_trace_err(J, LJ_TRERR_BADTYPE);
1360   return 0;
1361 }
1362 
1363 void LJ_FASTCALL recff_cdata_arith(jit_State *J, RecordFFData *rd)
1364 {
1365   CTState *cts = ctype_ctsG(J2G(J));
1366   TRef sp[2];
1367   CType *s[2];
1368   MSize i;
1369   for (i = 0; i < 2; i++) {
1370     TRef tr = J->base[i];
1371     CType *ct = ctype_get(cts, CTID_DOUBLE);
1372     if (!tr) {
1373       lj_trace_err(J, LJ_TRERR_BADTYPE);
1374     } else if (tref_iscdata(tr)) {
1375       CTypeID id = argv2cdata(J, tr, &rd->argv[i])->ctypeid;
1376       IRType t;
1377       ct = ctype_raw(cts, id);
1378       t = crec_ct2irt(cts, ct);
1379       if (ctype_isptr(ct->info)) {  /* Resolve pointer or reference. */
1380         tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_PTR);
1381         if (ctype_isref(ct->info)) {
1382           ct = ctype_rawchild(cts, ct);
1383           t = crec_ct2irt(cts, ct);
1384         }
1385       } else if (t == IRT_I64 || t == IRT_U64) {
1386         tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT64);
1387         lj_needsplit(J);
1388         goto ok;
1389       } else if (t == IRT_INT || t == IRT_U32) {
1390         tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT);
1391         if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1392         goto ok;
1393       } else if (ctype_isfunc(ct->info)) {
1394         tr = emitir(IRT(IR_FLOAD, IRT_PTR), tr, IRFL_CDATA_PTR);
1395         ct = ctype_get(cts,
1396           lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR));
1397         goto ok;
1398       } else {
1399         tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCcdata)));
1400       }
1401       if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1402       if (ctype_isnum(ct->info)) {
1403         if (t == IRT_CDATA) {
1404           tr = 0;
1405         } else {
1406           if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1407           tr = emitir(IRT(IR_XLOAD, t), tr, 0);
1408         }
1409       }
1410     } else if (tref_isnil(tr)) {
1411       tr = lj_ir_kptr(J, NULL);
1412       ct = ctype_get(cts, CTID_P_VOID);
1413     } else if (tref_isinteger(tr)) {
1414       ct = ctype_get(cts, CTID_INT32);
1415     } else if (tref_isstr(tr)) {
1416       TRef tr2 = J->base[1-i];
1417       CTypeID id = argv2cdata(J, tr2, &rd->argv[1-i])->ctypeid;
1418       ct = ctype_raw(cts, id);
1419       if (ctype_isenum(ct->info)) {  /* Match string against enum constant. */
1420         GCstr *str = strV(&rd->argv[i]);
1421         CTSize ofs;
1422         CType *cct = lj_ctype_getfield(cts, ct, str, &ofs);
1423         if (cct && ctype_isconstval(cct->info)) {
1424           /* Specialize to the name of the enum constant. */
1425           emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, str));
1426           ct = ctype_child(cts, cct);
1427           tr = lj_ir_kint(J, (int32_t)ofs);
1428         } else {  /* Interpreter will throw or return false. */
1429           ct = ctype_get(cts, CTID_P_VOID);
1430         }
1431       } else if (ctype_isptr(ct->info)) {
1432         tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCstr)));
1433       } else {
1434         ct = ctype_get(cts, CTID_P_VOID);
1435       }
1436     } else if (!tref_isnum(tr)) {
1437       tr = 0;
1438       ct = ctype_get(cts, CTID_P_VOID);
1439     }
1440   ok:
1441     s[i] = ct;
1442     sp[i] = tr;
1443   }
1444   {
1445     TRef tr;
1446     if (!(tr = crec_arith_int64(J, sp, s, (MMS)rd->data)) &&
1447         !(tr = crec_arith_ptr(J, sp, s, (MMS)rd->data)) &&
1448         !(tr = crec_arith_meta(J, sp, s, cts, rd)))
1449       return;
1450     J->base[0] = tr;
1451     /* Fixup cdata comparisons, too. Avoids some cdata escapes. */
1452     if (J->postproc == LJ_POST_FIXGUARD && frame_iscont(J->L->base-1) &&
1453         !irt_isguard(J->guardemit)) {
1454       const BCIns *pc = frame_contpc(J->L->base-1) - 1;
1455       if (bc_op(*pc) <= BC_ISNEP) {
1456         setframe_pc(&J2G(J)->tmptv, pc);
1457         J2G(J)->tmptv.u32.lo = ((tref_istrue(tr) ^ bc_op(*pc)) & 1);
1458         J->postproc = LJ_POST_FIXCOMP;
1459       }
1460     }
1461   }
1462 }
1463 
1464 /* -- C library namespace metamethods ------------------------------------- */
1465 
1466 void LJ_FASTCALL recff_clib_index(jit_State *J, RecordFFData *rd)
1467 {
1468   CTState *cts = ctype_ctsG(J2G(J));
1469   if (tref_isudata(J->base[0]) && tref_isstr(J->base[1]) &&
1470       udataV(&rd->argv[0])->udtype == UDTYPE_FFI_CLIB) {
1471     CLibrary *cl = (CLibrary *)uddata(udataV(&rd->argv[0]));
1472     GCstr *name = strV(&rd->argv[1]);
1473     CType *ct;
1474     CTypeID id = lj_ctype_getname(cts, &ct, name, CLNS_INDEX);
1475     cTValue *tv = lj_tab_getstr(cl->cache, name);
1476     rd->nres = rd->data;
1477     if (id && tv && !tvisnil(tv)) {
1478       /* Specialize to the symbol name and make the result a constant. */
1479       emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, name));
1480       if (ctype_isconstval(ct->info)) {
1481         if (ct->size >= 0x80000000u &&
1482             (ctype_child(cts, ct)->info & CTF_UNSIGNED))
1483           J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)ct->size);
1484         else
1485           J->base[0] = lj_ir_kint(J, (int32_t)ct->size);
1486       } else if (ctype_isextern(ct->info)) {
1487         CTypeID sid = ctype_cid(ct->info);
1488         void *sp = *(void **)cdataptr(cdataV(tv));
1489         TRef ptr;
1490         ct = ctype_raw(cts, sid);
1491         if (LJ_64 && !checkptr32(sp))
1492           ptr = lj_ir_kintp(J, (uintptr_t)sp);
1493         else
1494           ptr = lj_ir_kptr(J, sp);
1495         if (rd->data) {
1496           J->base[0] = crec_tv_ct(J, ct, sid, ptr);
1497         } else {
1498           J->needsnap = 1;
1499           crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
1500         }
1501       } else {
1502         J->base[0] = lj_ir_kgc(J, obj2gco(cdataV(tv)), IRT_CDATA);
1503       }
1504     } else {
1505       lj_trace_err(J, LJ_TRERR_NOCACHE);
1506     }
1507   }  /* else: interpreter will throw. */
1508 }
1509 
1510 /* -- FFI library functions ----------------------------------------------- */
1511 
1512 static TRef crec_toint(jit_State *J, CTState *cts, TRef sp, TValue *sval)
1513 {
1514   return crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, sp, sval);
1515 }
1516 
1517 void LJ_FASTCALL recff_ffi_new(jit_State *J, RecordFFData *rd)
1518 {
1519   crec_alloc(J, rd, argv2ctype(J, J->base[0], &rd->argv[0]));
1520 }
1521 
1522 void LJ_FASTCALL recff_ffi_errno(jit_State *J, RecordFFData *rd)
1523 {
1524   UNUSED(rd);
1525   if (J->base[0])
1526     lj_trace_err(J, LJ_TRERR_NYICALL);
1527   J->base[0] = lj_ir_call(J, IRCALL_lj_vm_errno);
1528 }
1529 
1530 void LJ_FASTCALL recff_ffi_string(jit_State *J, RecordFFData *rd)
1531 {
1532   CTState *cts = ctype_ctsG(J2G(J));
1533   TRef tr = J->base[0];
1534   if (tr) {
1535     TRef trlen = J->base[1];
1536     if (!tref_isnil(trlen)) {
1537       trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1538       tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, tr, &rd->argv[0]);
1539     } else {
1540       tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CCHAR), 0, tr, &rd->argv[0]);
1541       trlen = lj_ir_call(J, IRCALL_strlen, tr);
1542     }
1543     J->base[0] = emitir(IRT(IR_XSNEW, IRT_STR), tr, trlen);
1544   }  /* else: interpreter will throw. */
1545 }
1546 
1547 void LJ_FASTCALL recff_ffi_copy(jit_State *J, RecordFFData *rd)
1548 {
1549   CTState *cts = ctype_ctsG(J2G(J));
1550   TRef trdst = J->base[0], trsrc = J->base[1], trlen = J->base[2];
1551   if (trdst && trsrc && (trlen || tref_isstr(trsrc))) {
1552     trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1553     trsrc = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, trsrc, &rd->argv[1]);
1554     if (trlen) {
1555       trlen = crec_toint(J, cts, trlen, &rd->argv[2]);
1556     } else {
1557       trlen = emitir(IRTI(IR_FLOAD), J->base[1], IRFL_STR_LEN);
1558       trlen = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1));
1559     }
1560     rd->nres = 0;
1561     crec_copy(J, trdst, trsrc, trlen, NULL);
1562   }  /* else: interpreter will throw. */
1563 }
1564 
1565 void LJ_FASTCALL recff_ffi_fill(jit_State *J, RecordFFData *rd)
1566 {
1567   CTState *cts = ctype_ctsG(J2G(J));
1568   TRef trdst = J->base[0], trlen = J->base[1], trfill = J->base[2];
1569   if (trdst && trlen) {
1570     CTSize step = 1;
1571     if (tviscdata(&rd->argv[0])) {  /* Get alignment of original destination. */
1572       CTSize sz;
1573       CType *ct = ctype_raw(cts, cdataV(&rd->argv[0])->ctypeid);
1574       if (ctype_isptr(ct->info))
1575         ct = ctype_rawchild(cts, ct);
1576       step = (1u<<ctype_align(lj_ctype_info(cts, ctype_typeid(cts, ct), &sz)));
1577     }
1578     trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1579     trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1580     if (trfill)
1581       trfill = crec_toint(J, cts, trfill, &rd->argv[2]);
1582     else
1583       trfill = lj_ir_kint(J, 0);
1584     rd->nres = 0;
1585     crec_fill(J, trdst, trlen, trfill, step);
1586   }  /* else: interpreter will throw. */
1587 }
1588 
1589 void LJ_FASTCALL recff_ffi_typeof(jit_State *J, RecordFFData *rd)
1590 {
1591   if (tref_iscdata(J->base[0])) {
1592     TRef trid = lj_ir_kint(J, argv2ctype(J, J->base[0], &rd->argv[0]));
1593     J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA),
1594                         lj_ir_kint(J, CTID_CTYPEID), trid);
1595   } else {
1596     setfuncV(J->L, &J->errinfo, J->fn);
1597     lj_trace_err_info(J, LJ_TRERR_NYIFFU);
1598   }
1599 }
1600 
1601 void LJ_FASTCALL recff_ffi_istype(jit_State *J, RecordFFData *rd)
1602 {
1603   argv2ctype(J, J->base[0], &rd->argv[0]);
1604   if (tref_iscdata(J->base[1])) {
1605     argv2ctype(J, J->base[1], &rd->argv[1]);
1606     J->postproc = LJ_POST_FIXBOOL;
1607     J->base[0] = TREF_TRUE;
1608   } else {
1609     J->base[0] = TREF_FALSE;
1610   }
1611 }
1612 
1613 void LJ_FASTCALL recff_ffi_abi(jit_State *J, RecordFFData *rd)
1614 {
1615   if (tref_isstr(J->base[0])) {
1616     /* Specialize to the ABI string to make the boolean result a constant. */
1617     emitir(IRTG(IR_EQ, IRT_STR), J->base[0], lj_ir_kstr(J, strV(&rd->argv[0])));
1618     J->postproc = LJ_POST_FIXBOOL;
1619     J->base[0] = TREF_TRUE;
1620   } else {
1621     lj_trace_err(J, LJ_TRERR_BADTYPE);
1622   }
1623 }
1624 
1625 /* Record ffi.sizeof(), ffi.alignof(), ffi.offsetof(). */
1626 void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd)
1627 {
1628   CTypeID id = argv2ctype(J, J->base[0], &rd->argv[0]);
1629   if (rd->data == FF_ffi_sizeof) {
1630     CType *ct = lj_ctype_rawref(ctype_ctsG(J2G(J)), id);
1631     if (ctype_isvltype(ct->info))
1632       lj_trace_err(J, LJ_TRERR_BADTYPE);
1633   } else if (rd->data == FF_ffi_offsetof) {  /* Specialize to the field name. */
1634     if (!tref_isstr(J->base[1]))
1635       lj_trace_err(J, LJ_TRERR_BADTYPE);
1636     emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
1637     rd->nres = 3;  /* Just in case. */
1638   }
1639   J->postproc = LJ_POST_FIXCONST;
1640   J->base[0] = J->base[1] = J->base[2] = TREF_NIL;
1641 }
1642 
1643 void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd)
1644 {
1645   argv2cdata(J, J->base[0], &rd->argv[0]);
1646   crec_finalizer(J, J->base[0], &rd->argv[1]);
1647 }
1648 
1649 /* -- Miscellaneous library functions ------------------------------------- */
1650 
1651 void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd)
1652 {
1653   CTState *cts = ctype_ctsG(J2G(J));
1654   CType *d, *ct = lj_ctype_rawref(cts, cdataV(&rd->argv[0])->ctypeid);
1655   if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1656   if (ctype_isnum(ct->info) || ctype_iscomplex(ct->info)) {
1657     if (ctype_isinteger_or_bool(ct->info) && ct->size <= 4 &&
1658         !(ct->size == 4 && (ct->info & CTF_UNSIGNED)))
1659       d = ctype_get(cts, CTID_INT32);
1660     else
1661       d = ctype_get(cts, CTID_DOUBLE);
1662     J->base[0] = crec_ct_tv(J, d, 0, J->base[0], &rd->argv[0]);
1663   } else {
1664     J->base[0] = TREF_NIL;
1665   }
1666 }
1667 
1668 #undef IR
1669 #undef emitir
1670 #undef emitconv
1671 
1672 #endif

/* [<][>][^][v][top][bottom][index][help] */