root/lj_ccallback.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. CALLBACK_OFS2SLOT
  2. callback_slot2ptr
  3. lj_ccallback_ptr2slot
  4. callback_mcode_init
  5. callback_mcode_init
  6. callback_mcode_init
  7. callback_mcode_init
  8. callback_mcode_init
  9. callback_mcode_new
  10. lj_ccallback_mcode_free
  11. callback_conv_args
  12. callback_conv_result
  13. lj_ccallback_enter
  14. lj_ccallback_leave
  15. callback_slot_new
  16. callback_checkfunc
  17. lj_ccallback_new

   1 /*
   2 ** FFI C callback handling.
   3 ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5 
   6 #include "lj_obj.h"
   7 
   8 #if LJ_HASFFI
   9 
  10 #include "lj_gc.h"
  11 #include "lj_err.h"
  12 #include "lj_tab.h"
  13 #include "lj_state.h"
  14 #include "lj_frame.h"
  15 #include "lj_ctype.h"
  16 #include "lj_cconv.h"
  17 #include "lj_ccall.h"
  18 #include "lj_ccallback.h"
  19 #include "lj_target.h"
  20 #include "lj_mcode.h"
  21 #include "lj_trace.h"
  22 #include "lj_vm.h"
  23 
  24 /* -- Target-specific handling of callback slots -------------------------- */
  25 
  26 #define CALLBACK_MCODE_SIZE     (LJ_PAGESIZE * LJ_NUM_CBPAGE)
  27 
  28 #if LJ_OS_NOJIT
  29 
  30 /* Callbacks disabled. */
  31 #define CALLBACK_SLOT2OFS(slot) (0*(slot))
  32 #define CALLBACK_OFS2SLOT(ofs)  (0*(ofs))
  33 #define CALLBACK_MAX_SLOT       0
  34 
  35 #elif LJ_TARGET_X86ORX64
  36 
  37 #define CALLBACK_MCODE_HEAD     (LJ_64 ? 8 : 0)
  38 #define CALLBACK_MCODE_GROUP    (-2+1+2+(LJ_GC64 ? 10 : 5)+(LJ_64 ? 6 : 5))
  39 
  40 #define CALLBACK_SLOT2OFS(slot) \
  41   (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot))
  42 
  43 static MSize CALLBACK_OFS2SLOT(MSize ofs)
  44 {
  45   MSize group;
  46   ofs -= CALLBACK_MCODE_HEAD;
  47   group = ofs / (32*4 + CALLBACK_MCODE_GROUP);
  48   return (ofs % (32*4 + CALLBACK_MCODE_GROUP))/4 + group*32;
  49 }
  50 
  51 #define CALLBACK_MAX_SLOT \
  52   (((CALLBACK_MCODE_SIZE-CALLBACK_MCODE_HEAD)/(CALLBACK_MCODE_GROUP+4*32))*32)
  53 
  54 #elif LJ_TARGET_ARM
  55 
  56 #define CALLBACK_MCODE_HEAD             32
  57 
  58 #elif LJ_TARGET_ARM64
  59 
  60 #define CALLBACK_MCODE_HEAD             32
  61 
  62 #elif LJ_TARGET_PPC
  63 
  64 #define CALLBACK_MCODE_HEAD             24
  65 
  66 #elif LJ_TARGET_MIPS32
  67 
  68 #define CALLBACK_MCODE_HEAD             20
  69 
  70 #elif LJ_TARGET_MIPS64
  71 
  72 #define CALLBACK_MCODE_HEAD             52
  73 
  74 #else
  75 
  76 /* Missing support for this architecture. */
  77 #define CALLBACK_SLOT2OFS(slot) (0*(slot))
  78 #define CALLBACK_OFS2SLOT(ofs)  (0*(ofs))
  79 #define CALLBACK_MAX_SLOT       0
  80 
  81 #endif
  82 
  83 #ifndef CALLBACK_SLOT2OFS
  84 #define CALLBACK_SLOT2OFS(slot)         (CALLBACK_MCODE_HEAD + 8*(slot))
  85 #define CALLBACK_OFS2SLOT(ofs)          (((ofs)-CALLBACK_MCODE_HEAD)/8)
  86 #define CALLBACK_MAX_SLOT               (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
  87 #endif
  88 
  89 /* Convert callback slot number to callback function pointer. */
  90 static void *callback_slot2ptr(CTState *cts, MSize slot)
  91 {
  92   return (uint8_t *)cts->cb.mcode + CALLBACK_SLOT2OFS(slot);
  93 }
  94 
  95 /* Convert callback function pointer to slot number. */
  96 MSize lj_ccallback_ptr2slot(CTState *cts, void *p)
  97 {
  98   uintptr_t ofs = (uintptr_t)((uint8_t *)p -(uint8_t *)cts->cb.mcode);
  99   if (ofs < CALLBACK_MCODE_SIZE) {
 100     MSize slot = CALLBACK_OFS2SLOT((MSize)ofs);
 101     if (CALLBACK_SLOT2OFS(slot) == (MSize)ofs)
 102       return slot;
 103   }
 104   return ~0u;  /* Not a known callback function pointer. */
 105 }
 106 
 107 /* Initialize machine code for callback function pointers. */
 108 #if LJ_OS_NOJIT
 109 /* Disabled callback support. */
 110 #define callback_mcode_init(g, p)       UNUSED(p)
 111 #elif LJ_TARGET_X86ORX64
 112 static void callback_mcode_init(global_State *g, uint8_t *page)
 113 {
 114   uint8_t *p = page;
 115   uint8_t *target = (uint8_t *)(void *)lj_vm_ffi_callback;
 116   MSize slot;
 117 #if LJ_64
 118   *(void **)p = target; p += 8;
 119 #endif
 120   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 121     /* mov al, slot; jmp group */
 122     *p++ = XI_MOVrib | RID_EAX; *p++ = (uint8_t)slot;
 123     if ((slot & 31) == 31 || slot == CALLBACK_MAX_SLOT-1) {
 124       /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */
 125       *p++ = XI_PUSH + RID_EBP;
 126       *p++ = XI_MOVrib | (RID_EAX+4); *p++ = (uint8_t)(slot >> 8);
 127 #if LJ_GC64
 128       *p++ = 0x48; *p++ = XI_MOVri | RID_EBP;
 129       *(uint64_t *)p = (uint64_t)(g); p += 8;
 130 #else
 131       *p++ = XI_MOVri | RID_EBP;
 132       *(int32_t *)p = i32ptr(g); p += 4;
 133 #endif
 134 #if LJ_64
 135       /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */
 136       *p++ = XI_GROUP5; *p++ = XM_OFS0 + (XOg_JMP<<3) + RID_EBP;
 137       *(int32_t *)p = (int32_t)(page-(p+4)); p += 4;
 138 #else
 139       /* jmp lj_vm_ffi_callback. */
 140       *p++ = XI_JMP; *(int32_t *)p = target-(p+4); p += 4;
 141 #endif
 142     } else {
 143       *p++ = XI_JMPs; *p++ = (uint8_t)((2+2)*(31-(slot&31)) - 2);
 144     }
 145   }
 146   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 147 }
 148 #elif LJ_TARGET_ARM
 149 static void callback_mcode_init(global_State *g, uint32_t *page)
 150 {
 151   uint32_t *p = page;
 152   void *target = (void *)lj_vm_ffi_callback;
 153   MSize slot;
 154   /* This must match with the saveregs macro in buildvm_arm.dasc. */
 155   *p++ = ARMI_SUB|ARMF_D(RID_R12)|ARMF_N(RID_R12)|ARMF_M(RID_PC);
 156   *p++ = ARMI_PUSH|ARMF_N(RID_SP)|RSET_RANGE(RID_R4,RID_R11+1)|RID2RSET(RID_LR);
 157   *p++ = ARMI_SUB|ARMI_K12|ARMF_D(RID_R12)|ARMF_N(RID_R12)|CALLBACK_MCODE_HEAD;
 158   *p++ = ARMI_STR|ARMI_LS_P|ARMI_LS_W|ARMF_D(RID_R12)|ARMF_N(RID_SP)|(CFRAME_SIZE-4*9);
 159   *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_R12)|ARMF_N(RID_PC);
 160   *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_PC)|ARMF_N(RID_PC);
 161   *p++ = u32ptr(g);
 162   *p++ = u32ptr(target);
 163   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 164     *p++ = ARMI_MOV|ARMF_D(RID_R12)|ARMF_M(RID_PC);
 165     *p = ARMI_B | ((page-p-2) & 0x00ffffffu);
 166     p++;
 167   }
 168   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 169 }
 170 #elif LJ_TARGET_ARM64
 171 static void callback_mcode_init(global_State *g, uint32_t *page)
 172 {
 173   uint32_t *p = page;
 174   void *target = (void *)lj_vm_ffi_callback;
 175   MSize slot;
 176   *p++ = A64I_LE(A64I_LDRLx | A64F_D(RID_X11) | A64F_S19(4));
 177   *p++ = A64I_LE(A64I_LDRLx | A64F_D(RID_X10) | A64F_S19(5));
 178   *p++ = A64I_LE(A64I_BR | A64F_N(RID_X11));
 179   *p++ = A64I_LE(A64I_NOP);
 180   ((void **)p)[0] = target;
 181   ((void **)p)[1] = g;
 182   p += 4;
 183   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 184     *p++ = A64I_LE(A64I_MOVZw | A64F_D(RID_X9) | A64F_U16(slot));
 185     *p = A64I_LE(A64I_B | A64F_S26((page-p) & 0x03ffffffu));
 186     p++;
 187   }
 188   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 189 }
 190 #elif LJ_TARGET_PPC
 191 static void callback_mcode_init(global_State *g, uint32_t *page)
 192 {
 193   uint32_t *p = page;
 194   void *target = (void *)lj_vm_ffi_callback;
 195   MSize slot;
 196   *p++ = PPCI_LIS | PPCF_T(RID_TMP) | (u32ptr(target) >> 16);
 197   *p++ = PPCI_LIS | PPCF_T(RID_R12) | (u32ptr(g) >> 16);
 198   *p++ = PPCI_ORI | PPCF_A(RID_TMP)|PPCF_T(RID_TMP) | (u32ptr(target) & 0xffff);
 199   *p++ = PPCI_ORI | PPCF_A(RID_R12)|PPCF_T(RID_R12) | (u32ptr(g) & 0xffff);
 200   *p++ = PPCI_MTCTR | PPCF_T(RID_TMP);
 201   *p++ = PPCI_BCTR;
 202   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 203     *p++ = PPCI_LI | PPCF_T(RID_R11) | slot;
 204     *p = PPCI_B | (((page-p) & 0x00ffffffu) << 2);
 205     p++;
 206   }
 207   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 208 }
 209 #elif LJ_TARGET_MIPS
 210 static void callback_mcode_init(global_State *g, uint32_t *page)
 211 {
 212   uint32_t *p = page;
 213   uintptr_t target = (uintptr_t)(void *)lj_vm_ffi_callback;
 214   uintptr_t ug = (uintptr_t)(void *)g;
 215   MSize slot;
 216 #if LJ_TARGET_MIPS32
 217   *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 16);
 218   *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 16);
 219 #else
 220   *p++ = MIPSI_LUI  | MIPSF_T(RID_R3) | (target >> 48);
 221   *p++ = MIPSI_LUI  | MIPSF_T(RID_R2) | (ug >> 48);
 222   *p++ = MIPSI_ORI  | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 32) & 0xffff);
 223   *p++ = MIPSI_ORI  | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 32) & 0xffff);
 224   *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16);
 225   *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16);
 226   *p++ = MIPSI_ORI  | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 16) & 0xffff);
 227   *p++ = MIPSI_ORI  | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 16) & 0xffff);
 228   *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16);
 229   *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16);
 230 #endif
 231   *p++ = MIPSI_ORI  | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | (target & 0xffff);
 232   *p++ = MIPSI_JR | MIPSF_S(RID_R3);
 233   *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | (ug & 0xffff);
 234   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 235     *p = MIPSI_B | ((page-p-1) & 0x0000ffffu);
 236     p++;
 237     *p++ = MIPSI_LI | MIPSF_T(RID_R1) | slot;
 238   }
 239   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 240 }
 241 #else
 242 /* Missing support for this architecture. */
 243 #define callback_mcode_init(g, p)       UNUSED(p)
 244 #endif
 245 
 246 /* -- Machine code management --------------------------------------------- */
 247 
 248 #if LJ_TARGET_WINDOWS
 249 
 250 #define WIN32_LEAN_AND_MEAN
 251 #include <windows.h>
 252 
 253 #elif LJ_TARGET_POSIX
 254 
 255 #include <sys/mman.h>
 256 #ifndef MAP_ANONYMOUS
 257 #define MAP_ANONYMOUS   MAP_ANON
 258 #endif
 259 
 260 #endif
 261 
 262 /* Allocate and initialize area for callback function pointers. */
 263 static void callback_mcode_new(CTState *cts)
 264 {
 265   size_t sz = (size_t)CALLBACK_MCODE_SIZE;
 266   void *p;
 267   if (CALLBACK_MAX_SLOT == 0)
 268     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
 269 #if LJ_TARGET_WINDOWS
 270   p = VirtualAlloc(NULL, sz, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
 271   if (!p)
 272     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
 273 #elif LJ_TARGET_POSIX
 274   p = mmap(NULL, sz, (PROT_READ|PROT_WRITE), MAP_PRIVATE|MAP_ANONYMOUS,
 275            -1, 0);
 276   if (p == MAP_FAILED)
 277     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
 278 #else
 279   /* Fallback allocator. Fails if memory is not executable by default. */
 280   p = lj_mem_new(cts->L, sz);
 281 #endif
 282   cts->cb.mcode = p;
 283   callback_mcode_init(cts->g, p);
 284   lj_mcode_sync(p, (char *)p + sz);
 285 #if LJ_TARGET_WINDOWS
 286   {
 287     DWORD oprot;
 288     VirtualProtect(p, sz, PAGE_EXECUTE_READ, &oprot);
 289   }
 290 #elif LJ_TARGET_POSIX
 291   mprotect(p, sz, (PROT_READ|PROT_EXEC));
 292 #endif
 293 }
 294 
 295 /* Free area for callback function pointers. */
 296 void lj_ccallback_mcode_free(CTState *cts)
 297 {
 298   size_t sz = (size_t)CALLBACK_MCODE_SIZE;
 299   void *p = cts->cb.mcode;
 300   if (p == NULL) return;
 301 #if LJ_TARGET_WINDOWS
 302   VirtualFree(p, 0, MEM_RELEASE);
 303   UNUSED(sz);
 304 #elif LJ_TARGET_POSIX
 305   munmap(p, sz);
 306 #else
 307   lj_mem_free(cts->g, p, sz);
 308 #endif
 309 }
 310 
 311 /* -- C callback entry ---------------------------------------------------- */
 312 
 313 /* Target-specific handling of register arguments. Similar to lj_ccall.c. */
 314 #if LJ_TARGET_X86
 315 
 316 #define CALLBACK_HANDLE_REGARG \
 317   if (!isfp) {  /* Only non-FP values may be passed in registers. */ \
 318     if (n > 1) {  /* Anything > 32 bit is passed on the stack. */ \
 319       if (!LJ_ABI_WIN) ngpr = maxgpr;  /* Prevent reordering. */ \
 320     } else if (ngpr + 1 <= maxgpr) { \
 321       sp = &cts->cb.gpr[ngpr]; \
 322       ngpr += n; \
 323       goto done; \
 324     } \
 325   }
 326 
 327 #elif LJ_TARGET_X64 && LJ_ABI_WIN
 328 
 329 /* Windows/x64 argument registers are strictly positional (use ngpr). */
 330 #define CALLBACK_HANDLE_REGARG \
 331   if (isfp) { \
 332     if (ngpr < maxgpr) { sp = &cts->cb.fpr[ngpr++]; UNUSED(nfpr); goto done; } \
 333   } else { \
 334     if (ngpr < maxgpr) { sp = &cts->cb.gpr[ngpr++]; goto done; } \
 335   }
 336 
 337 #elif LJ_TARGET_X64
 338 
 339 #define CALLBACK_HANDLE_REGARG \
 340   if (isfp) { \
 341     if (nfpr + n <= CCALL_NARG_FPR) { \
 342       sp = &cts->cb.fpr[nfpr]; \
 343       nfpr += n; \
 344       goto done; \
 345     } \
 346   } else { \
 347     if (ngpr + n <= maxgpr) { \
 348       sp = &cts->cb.gpr[ngpr]; \
 349       ngpr += n; \
 350       goto done; \
 351     } \
 352   }
 353 
 354 #elif LJ_TARGET_ARM
 355 
 356 #if LJ_ABI_SOFTFP
 357 
 358 #define CALLBACK_HANDLE_REGARG_FP1      UNUSED(isfp);
 359 #define CALLBACK_HANDLE_REGARG_FP2
 360 
 361 #else
 362 
 363 #define CALLBACK_HANDLE_REGARG_FP1 \
 364   if (isfp) { \
 365     if (n == 1) { \
 366       if (fprodd) { \
 367         sp = &cts->cb.fpr[fprodd-1]; \
 368         fprodd = 0; \
 369         goto done; \
 370       } else if (nfpr + 1 <= CCALL_NARG_FPR) { \
 371         sp = &cts->cb.fpr[nfpr++]; \
 372         fprodd = nfpr; \
 373         goto done; \
 374       } \
 375     } else { \
 376       if (nfpr + 1 <= CCALL_NARG_FPR) { \
 377         sp = &cts->cb.fpr[nfpr++]; \
 378         goto done; \
 379       } \
 380     } \
 381     fprodd = 0;  /* No reordering after the first FP value is on stack. */ \
 382   } else {
 383 
 384 #define CALLBACK_HANDLE_REGARG_FP2      }
 385 
 386 #endif
 387 
 388 #define CALLBACK_HANDLE_REGARG \
 389   CALLBACK_HANDLE_REGARG_FP1 \
 390   if (n > 1) ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 391   if (ngpr + n <= maxgpr) { \
 392     sp = &cts->cb.gpr[ngpr]; \
 393     ngpr += n; \
 394     goto done; \
 395   } CALLBACK_HANDLE_REGARG_FP2
 396 
 397 #elif LJ_TARGET_ARM64
 398 
 399 #define CALLBACK_HANDLE_REGARG \
 400   if (isfp) { \
 401     if (nfpr + n <= CCALL_NARG_FPR) { \
 402       sp = &cts->cb.fpr[nfpr]; \
 403       nfpr += n; \
 404       goto done; \
 405     } else { \
 406       nfpr = CCALL_NARG_FPR;  /* Prevent reordering. */ \
 407     } \
 408   } else { \
 409     if (!LJ_TARGET_IOS && n > 1) \
 410       ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 411     if (ngpr + n <= maxgpr) { \
 412       sp = &cts->cb.gpr[ngpr]; \
 413       ngpr += n; \
 414       goto done; \
 415     } else { \
 416       ngpr = CCALL_NARG_GPR;  /* Prevent reordering. */ \
 417     } \
 418   }
 419 
 420 #elif LJ_TARGET_PPC
 421 
 422 #define CALLBACK_HANDLE_GPR \
 423   if (n > 1) { \
 424     lua_assert(((LJ_ABI_SOFTFP && ctype_isnum(cta->info)) ||  /* double. */ \
 425                 ctype_isinteger(cta->info)) && n == 2);  /* int64_t. */ \
 426     ngpr = (ngpr + 1u) & ~1u;  /* Align int64_t to regpair. */ \
 427   } \
 428   if (ngpr + n <= maxgpr) { \
 429     sp = &cts->cb.gpr[ngpr]; \
 430     ngpr += n; \
 431     goto done; \
 432   }
 433 
 434 #if LJ_ABI_SOFTFP
 435 #define CALLBACK_HANDLE_REGARG \
 436   CALLBACK_HANDLE_GPR \
 437   UNUSED(isfp);
 438 #else
 439 #define CALLBACK_HANDLE_REGARG \
 440   if (isfp) { \
 441     if (nfpr + 1 <= CCALL_NARG_FPR) { \
 442       sp = &cts->cb.fpr[nfpr++]; \
 443       cta = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */ \
 444       goto done; \
 445     } \
 446   } else {  /* Try to pass argument in GPRs. */ \
 447     CALLBACK_HANDLE_GPR \
 448   }
 449 #endif
 450 
 451 #if !LJ_ABI_SOFTFP
 452 #define CALLBACK_HANDLE_RET \
 453   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 454     *(double *)dp = *(float *)dp;  /* FPRs always hold doubles. */
 455 #endif
 456 
 457 #elif LJ_TARGET_MIPS32
 458 
 459 #define CALLBACK_HANDLE_GPR \
 460   if (n > 1) ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 461   if (ngpr + n <= maxgpr) { \
 462     sp = &cts->cb.gpr[ngpr]; \
 463     ngpr += n; \
 464     goto done; \
 465   }
 466 
 467 #if !LJ_ABI_SOFTFP      /* MIPS32 hard-float */
 468 #define CALLBACK_HANDLE_REGARG \
 469   if (isfp && nfpr < CCALL_NARG_FPR) {  /* Try to pass argument in FPRs. */ \
 470     sp = (void *)((uint8_t *)&cts->cb.fpr[nfpr] + ((LJ_BE && n==1) ? 4 : 0)); \
 471     nfpr++; ngpr += n; \
 472     goto done; \
 473   } else {  /* Try to pass argument in GPRs. */ \
 474     nfpr = CCALL_NARG_FPR; \
 475     CALLBACK_HANDLE_GPR \
 476   }
 477 #else                   /* MIPS32 soft-float */
 478 #define CALLBACK_HANDLE_REGARG \
 479   CALLBACK_HANDLE_GPR \
 480   UNUSED(isfp);
 481 #endif
 482 
 483 #define CALLBACK_HANDLE_RET \
 484   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 485     ((float *)dp)[1] = *(float *)dp;
 486 
 487 #elif LJ_TARGET_MIPS64
 488 
 489 #if !LJ_ABI_SOFTFP      /* MIPS64 hard-float */
 490 #define CALLBACK_HANDLE_REGARG \
 491   if (ngpr + n <= maxgpr) { \
 492     sp = isfp ? (void*) &cts->cb.fpr[ngpr] : (void*) &cts->cb.gpr[ngpr]; \
 493     ngpr += n; \
 494     goto done; \
 495   }
 496 #else                   /* MIPS64 soft-float */
 497 #define CALLBACK_HANDLE_REGARG \
 498   if (ngpr + n <= maxgpr) { \
 499     UNUSED(isfp); \
 500     sp = (void*) &cts->cb.gpr[ngpr]; \
 501     ngpr += n; \
 502     goto done; \
 503   }
 504 #endif
 505 
 506 #define CALLBACK_HANDLE_RET \
 507   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 508     ((float *)dp)[1] = *(float *)dp;
 509 
 510 #else
 511 #error "Missing calling convention definitions for this architecture"
 512 #endif
 513 
 514 /* Convert and push callback arguments to Lua stack. */
 515 static void callback_conv_args(CTState *cts, lua_State *L)
 516 {
 517   TValue *o = L->top;
 518   intptr_t *stack = cts->cb.stack;
 519   MSize slot = cts->cb.slot;
 520   CTypeID id = 0, rid, fid;
 521   int gcsteps = 0;
 522   CType *ct;
 523   GCfunc *fn;
 524   int fntp;
 525   MSize ngpr = 0, nsp = 0, maxgpr = CCALL_NARG_GPR;
 526 #if CCALL_NARG_FPR
 527   MSize nfpr = 0;
 528 #if LJ_TARGET_ARM
 529   MSize fprodd = 0;
 530 #endif
 531 #endif
 532 
 533   if (slot < cts->cb.sizeid && (id = cts->cb.cbid[slot]) != 0) {
 534     ct = ctype_get(cts, id);
 535     rid = ctype_cid(ct->info);  /* Return type. x86: +(spadj<<16). */
 536     fn = funcV(lj_tab_getint(cts->miscmap, (int32_t)slot));
 537     fntp = LJ_TFUNC;
 538   } else {  /* Must set up frame first, before throwing the error. */
 539     ct = NULL;
 540     rid = 0;
 541     fn = (GCfunc *)L;
 542     fntp = LJ_TTHREAD;
 543   }
 544   /* Continuation returns from callback. */
 545   if (LJ_FR2) {
 546     (o++)->u64 = LJ_CONT_FFI_CALLBACK;
 547     (o++)->u64 = rid;
 548     o++;
 549   } else {
 550     o->u32.lo = LJ_CONT_FFI_CALLBACK;
 551     o->u32.hi = rid;
 552     o++;
 553   }
 554   setframe_gc(o, obj2gco(fn), fntp);
 555   setframe_ftsz(o, ((char *)(o+1) - (char *)L->base) + FRAME_CONT);
 556   L->top = L->base = ++o;
 557   if (!ct)
 558     lj_err_caller(cts->L, LJ_ERR_FFI_BADCBACK);
 559   if (isluafunc(fn))
 560     setcframe_pc(L->cframe, proto_bc(funcproto(fn))+1);
 561   lj_state_checkstack(L, LUA_MINSTACK);  /* May throw. */
 562   o = L->base;  /* Might have been reallocated. */
 563 
 564 #if LJ_TARGET_X86
 565   /* x86 has several different calling conventions. */
 566   switch (ctype_cconv(ct->info)) {
 567   case CTCC_FASTCALL: maxgpr = 2; break;
 568   case CTCC_THISCALL: maxgpr = 1; break;
 569   default: maxgpr = 0; break;
 570   }
 571 #endif
 572 
 573   fid = ct->sib;
 574   while (fid) {
 575     CType *ctf = ctype_get(cts, fid);
 576     if (!ctype_isattrib(ctf->info)) {
 577       CType *cta;
 578       void *sp;
 579       CTSize sz;
 580       int isfp;
 581       MSize n;
 582       lua_assert(ctype_isfield(ctf->info));
 583       cta = ctype_rawchild(cts, ctf);
 584       isfp = ctype_isfp(cta->info);
 585       sz = (cta->size + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
 586       n = sz / CTSIZE_PTR;  /* Number of GPRs or stack slots needed. */
 587 
 588       CALLBACK_HANDLE_REGARG  /* Handle register arguments. */
 589 
 590       /* Otherwise pass argument on stack. */
 591       if (CCALL_ALIGN_STACKARG && LJ_32 && sz == 8)
 592         nsp = (nsp + 1) & ~1u;  /* Align 64 bit argument on stack. */
 593       sp = &stack[nsp];
 594       nsp += n;
 595 
 596     done:
 597       if (LJ_BE && cta->size < CTSIZE_PTR
 598 #if LJ_TARGET_MIPS64
 599           && !(isfp && nsp)
 600 #endif
 601          )
 602         sp = (void *)((uint8_t *)sp + CTSIZE_PTR-cta->size);
 603       gcsteps += lj_cconv_tv_ct(cts, cta, 0, o++, sp);
 604     }
 605     fid = ctf->sib;
 606   }
 607   L->top = o;
 608 #if LJ_TARGET_X86
 609   /* Store stack adjustment for returns from non-cdecl callbacks. */
 610   if (ctype_cconv(ct->info) != CTCC_CDECL) {
 611 #if LJ_FR2
 612     (L->base-3)->u64 |= (nsp << (16+2));
 613 #else
 614     (L->base-2)->u32.hi |= (nsp << (16+2));
 615 #endif
 616   }
 617 #endif
 618   while (gcsteps-- > 0)
 619     lj_gc_check(L);
 620 }
 621 
 622 /* Convert Lua object to callback result. */
 623 static void callback_conv_result(CTState *cts, lua_State *L, TValue *o)
 624 {
 625 #if LJ_FR2
 626   CType *ctr = ctype_raw(cts, (uint16_t)(L->base-3)->u64);
 627 #else
 628   CType *ctr = ctype_raw(cts, (uint16_t)(L->base-2)->u32.hi);
 629 #endif
 630 #if LJ_TARGET_X86
 631   cts->cb.gpr[2] = 0;
 632 #endif
 633   if (!ctype_isvoid(ctr->info)) {
 634     uint8_t *dp = (uint8_t *)&cts->cb.gpr[0];
 635 #if CCALL_NUM_FPR
 636     if (ctype_isfp(ctr->info))
 637       dp = (uint8_t *)&cts->cb.fpr[0];
 638 #endif
 639 #if LJ_TARGET_ARM64 && LJ_BE
 640     if (ctype_isfp(ctr->info) && ctr->size == sizeof(float))
 641       dp = (uint8_t *)&cts->cb.fpr[0].f[1];
 642 #endif
 643     lj_cconv_ct_tv(cts, ctr, dp, o, 0);
 644 #ifdef CALLBACK_HANDLE_RET
 645     CALLBACK_HANDLE_RET
 646 #endif
 647     /* Extend returned integers to (at least) 32 bits. */
 648     if (ctype_isinteger_or_bool(ctr->info) && ctr->size < 4) {
 649       if (ctr->info & CTF_UNSIGNED)
 650         *(uint32_t *)dp = ctr->size == 1 ? (uint32_t)*(uint8_t *)dp :
 651                                            (uint32_t)*(uint16_t *)dp;
 652       else
 653         *(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp :
 654                                           (int32_t)*(int16_t *)dp;
 655     }
 656 #if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
 657     /* Always sign-extend results to 64 bits. Even a soft-fp 'float'. */
 658     if (ctr->size <= 4 &&
 659         (LJ_ABI_SOFTFP || ctype_isinteger_or_bool(ctr->info)))
 660       *(int64_t *)dp = (int64_t)*(int32_t *)dp;
 661 #endif
 662 #if LJ_TARGET_X86
 663     if (ctype_isfp(ctr->info))
 664       cts->cb.gpr[2] = ctr->size == sizeof(float) ? 1 : 2;
 665 #endif
 666   }
 667 }
 668 
 669 /* Enter callback. */
 670 lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf)
 671 {
 672   lua_State *L = cts->L;
 673   global_State *g = cts->g;
 674   lua_assert(L != NULL);
 675   if (tvref(g->jit_base)) {
 676     setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK));
 677     if (g->panic) g->panic(L);
 678     exit(EXIT_FAILURE);
 679   }
 680   lj_trace_abort(g);  /* Never record across callback. */
 681   /* Setup C frame. */
 682   cframe_prev(cf) = L->cframe;
 683   setcframe_L(cf, L);
 684   cframe_errfunc(cf) = -1;
 685   cframe_nres(cf) = 0;
 686   L->cframe = cf;
 687   callback_conv_args(cts, L);
 688   return L;  /* Now call the function on this stack. */
 689 }
 690 
 691 /* Leave callback. */
 692 void LJ_FASTCALL lj_ccallback_leave(CTState *cts, TValue *o)
 693 {
 694   lua_State *L = cts->L;
 695   GCfunc *fn;
 696   TValue *obase = L->base;
 697   L->base = L->top;  /* Keep continuation frame for throwing errors. */
 698   if (o >= L->base) {
 699     /* PC of RET* is lost. Point to last line for result conv. errors. */
 700     fn = curr_func(L);
 701     if (isluafunc(fn)) {
 702       GCproto *pt = funcproto(fn);
 703       setcframe_pc(L->cframe, proto_bc(pt)+pt->sizebc+1);
 704     }
 705   }
 706   callback_conv_result(cts, L, o);
 707   /* Finally drop C frame and continuation frame. */
 708   L->top -= 2+2*LJ_FR2;
 709   L->base = obase;
 710   L->cframe = cframe_prev(L->cframe);
 711   cts->cb.slot = 0;  /* Blacklist C function that called the callback. */
 712 }
 713 
 714 /* -- C callback management ----------------------------------------------- */
 715 
 716 /* Get an unused slot in the callback slot table. */
 717 static MSize callback_slot_new(CTState *cts, CType *ct)
 718 {
 719   CTypeID id = ctype_typeid(cts, ct);
 720   CTypeID1 *cbid = cts->cb.cbid;
 721   MSize top;
 722   for (top = cts->cb.topid; top < cts->cb.sizeid; top++)
 723     if (LJ_LIKELY(cbid[top] == 0))
 724       goto found;
 725 #if CALLBACK_MAX_SLOT
 726   if (top >= CALLBACK_MAX_SLOT)
 727 #endif
 728     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
 729   if (!cts->cb.mcode)
 730     callback_mcode_new(cts);
 731   lj_mem_growvec(cts->L, cbid, cts->cb.sizeid, CALLBACK_MAX_SLOT, CTypeID1);
 732   cts->cb.cbid = cbid;
 733   memset(cbid+top, 0, (cts->cb.sizeid-top)*sizeof(CTypeID1));
 734 found:
 735   cbid[top] = id;
 736   cts->cb.topid = top+1;
 737   return top;
 738 }
 739 
 740 /* Check for function pointer and supported argument/result types. */
 741 static CType *callback_checkfunc(CTState *cts, CType *ct)
 742 {
 743   int narg = 0;
 744   if (!ctype_isptr(ct->info) || (LJ_64 && ct->size != CTSIZE_PTR))
 745     return NULL;
 746   ct = ctype_rawchild(cts, ct);
 747   if (ctype_isfunc(ct->info)) {
 748     CType *ctr = ctype_rawchild(cts, ct);
 749     CTypeID fid = ct->sib;
 750     if (!(ctype_isvoid(ctr->info) || ctype_isenum(ctr->info) ||
 751           ctype_isptr(ctr->info) || (ctype_isnum(ctr->info) && ctr->size <= 8)))
 752       return NULL;
 753     if ((ct->info & CTF_VARARG))
 754       return NULL;
 755     while (fid) {
 756       CType *ctf = ctype_get(cts, fid);
 757       if (!ctype_isattrib(ctf->info)) {
 758         CType *cta;
 759         lua_assert(ctype_isfield(ctf->info));
 760         cta = ctype_rawchild(cts, ctf);
 761         if (!(ctype_isenum(cta->info) || ctype_isptr(cta->info) ||
 762               (ctype_isnum(cta->info) && cta->size <= 8)) ||
 763             ++narg >= LUA_MINSTACK-3)
 764           return NULL;
 765       }
 766       fid = ctf->sib;
 767     }
 768     return ct;
 769   }
 770   return NULL;
 771 }
 772 
 773 /* Create a new callback and return the callback function pointer. */
 774 void *lj_ccallback_new(CTState *cts, CType *ct, GCfunc *fn)
 775 {
 776   ct = callback_checkfunc(cts, ct);
 777   if (ct) {
 778     MSize slot = callback_slot_new(cts, ct);
 779     GCtab *t = cts->miscmap;
 780     setfuncV(cts->L, lj_tab_setint(cts->L, t, (int32_t)slot), fn);
 781     lj_gc_anybarriert(cts->L, t);
 782     return callback_slot2ptr(cts, slot);
 783   }
 784   return NULL;  /* Bad conversion. */
 785 }
 786 
 787 #endif

/* [<][>][^][v][top][bottom][index][help] */