root/lj_lex.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. lex_more
  2. lex_next
  3. lex_save
  4. lex_savenext
  5. lex_newline
  6. lex_number
  7. lex_skipeq
  8. lex_longstring
  9. lex_string
  10. lex_scan
  11. lj_lex_setup
  12. lj_lex_cleanup
  13. lj_lex_next
  14. lj_lex_lookahead
  15. lj_lex_token2str
  16. lj_lex_error
  17. lj_lex_init

   1 /*
   2 ** Lexical analyzer.
   3 ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
   4 **
   5 ** Major portions taken verbatim or adapted from the Lua interpreter.
   6 ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
   7 */
   8 
   9 #define lj_lex_c
  10 #define LUA_CORE
  11 
  12 #include "lj_obj.h"
  13 #include "lj_gc.h"
  14 #include "lj_err.h"
  15 #include "lj_buf.h"
  16 #include "lj_str.h"
  17 #if LJ_HASFFI
  18 #include "lj_tab.h"
  19 #include "lj_ctype.h"
  20 #include "lj_cdata.h"
  21 #include "lualib.h"
  22 #endif
  23 #include "lj_state.h"
  24 #include "lj_lex.h"
  25 #include "lj_parse.h"
  26 #include "lj_char.h"
  27 #include "lj_strscan.h"
  28 #include "lj_strfmt.h"
  29 
  30 /* Lua lexer token names. */
  31 static const char *const tokennames[] = {
  32 #define TKSTR1(name)            #name,
  33 #define TKSTR2(name, sym)       #sym,
  34 TKDEF(TKSTR1, TKSTR2)
  35 #undef TKSTR1
  36 #undef TKSTR2
  37   NULL
  38 };
  39 
  40 /* -- Buffer handling ----------------------------------------------------- */
  41 
  42 #define LEX_EOF                 (-1)
  43 #define lex_iseol(ls)           (ls->c == '\n' || ls->c == '\r')
  44 
  45 /* Get more input from reader. */
  46 static LJ_NOINLINE LexChar lex_more(LexState *ls)
  47 {
  48   size_t sz;
  49   const char *p = ls->rfunc(ls->L, ls->rdata, &sz);
  50   if (p == NULL || sz == 0) return LEX_EOF;
  51   ls->pe = p + sz;
  52   ls->p = p + 1;
  53   return (LexChar)(uint8_t)p[0];
  54 }
  55 
  56 /* Get next character. */
  57 static LJ_AINLINE LexChar lex_next(LexState *ls)
  58 {
  59   return (ls->c = ls->p < ls->pe ? (LexChar)(uint8_t)*ls->p++ : lex_more(ls));
  60 }
  61 
  62 /* Save character. */
  63 static LJ_AINLINE void lex_save(LexState *ls, LexChar c)
  64 {
  65   lj_buf_putb(&ls->sb, c);
  66 }
  67 
  68 /* Save previous character and get next character. */
  69 static LJ_AINLINE LexChar lex_savenext(LexState *ls)
  70 {
  71   lex_save(ls, ls->c);
  72   return lex_next(ls);
  73 }
  74 
  75 /* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
  76 static void lex_newline(LexState *ls)
  77 {
  78   LexChar old = ls->c;
  79   lua_assert(lex_iseol(ls));
  80   lex_next(ls);  /* Skip "\n" or "\r". */
  81   if (lex_iseol(ls) && ls->c != old) lex_next(ls);  /* Skip "\n\r" or "\r\n". */
  82   if (++ls->linenumber >= LJ_MAX_LINE)
  83     lj_lex_error(ls, ls->tok, LJ_ERR_XLINES);
  84 }
  85 
  86 /* -- Scanner for terminals ----------------------------------------------- */
  87 
  88 /* Parse a number literal. */
  89 static void lex_number(LexState *ls, TValue *tv)
  90 {
  91   StrScanFmt fmt;
  92   LexChar c, xp = 'e';
  93   lua_assert(lj_char_isdigit(ls->c));
  94   if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x')
  95     xp = 'p';
  96   while (lj_char_isident(ls->c) || ls->c == '.' ||
  97          ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) {
  98     c = ls->c;
  99     lex_savenext(ls);
 100   }
 101   lex_save(ls, '\0');
 102   fmt = lj_strscan_scan((const uint8_t *)sbufB(&ls->sb), tv,
 103           (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) |
 104           (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0));
 105   if (LJ_DUALNUM && fmt == STRSCAN_INT) {
 106     setitype(tv, LJ_TISNUM);
 107   } else if (fmt == STRSCAN_NUM) {
 108     /* Already in correct format. */
 109 #if LJ_HASFFI
 110   } else if (fmt != STRSCAN_ERROR) {
 111     lua_State *L = ls->L;
 112     GCcdata *cd;
 113     lua_assert(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG);
 114     if (!ctype_ctsG(G(L))) {
 115       ptrdiff_t oldtop = savestack(L, L->top);
 116       luaopen_ffi(L);  /* Load FFI library on-demand. */
 117       L->top = restorestack(L, oldtop);
 118     }
 119     if (fmt == STRSCAN_IMAG) {
 120       cd = lj_cdata_new_(L, CTID_COMPLEX_DOUBLE, 2*sizeof(double));
 121       ((double *)cdataptr(cd))[0] = 0;
 122       ((double *)cdataptr(cd))[1] = numV(tv);
 123     } else {
 124       cd = lj_cdata_new_(L, fmt==STRSCAN_I64 ? CTID_INT64 : CTID_UINT64, 8);
 125       *(uint64_t *)cdataptr(cd) = tv->u64;
 126     }
 127     lj_parse_keepcdata(ls, tv, cd);
 128 #endif
 129   } else {
 130     lua_assert(fmt == STRSCAN_ERROR);
 131     lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER);
 132   }
 133 }
 134 
 135 /* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
 136 static int lex_skipeq(LexState *ls)
 137 {
 138   int count = 0;
 139   LexChar s = ls->c;
 140   lua_assert(s == '[' || s == ']');
 141   while (lex_savenext(ls) == '=')
 142     count++;
 143   return (ls->c == s) ? count : (-count) - 1;
 144 }
 145 
 146 /* Parse a long string or long comment (tv set to NULL). */
 147 static void lex_longstring(LexState *ls, TValue *tv, int sep)
 148 {
 149   lex_savenext(ls);  /* Skip second '['. */
 150   if (lex_iseol(ls))  /* Skip initial newline. */
 151     lex_newline(ls);
 152   for (;;) {
 153     switch (ls->c) {
 154     case LEX_EOF:
 155       lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
 156       break;
 157     case ']':
 158       if (lex_skipeq(ls) == sep) {
 159         lex_savenext(ls);  /* Skip second ']'. */
 160         goto endloop;
 161       }
 162       break;
 163     case '\n':
 164     case '\r':
 165       lex_save(ls, '\n');
 166       lex_newline(ls);
 167       if (!tv) lj_buf_reset(&ls->sb);  /* Don't waste space for comments. */
 168       break;
 169     default:
 170       lex_savenext(ls);
 171       break;
 172     }
 173   } endloop:
 174   if (tv) {
 175     GCstr *str = lj_parse_keepstr(ls, sbufB(&ls->sb) + (2 + (MSize)sep),
 176                                       sbuflen(&ls->sb) - 2*(2 + (MSize)sep));
 177     setstrV(ls->L, tv, str);
 178   }
 179 }
 180 
 181 /* Parse a string. */
 182 static void lex_string(LexState *ls, TValue *tv)
 183 {
 184   LexChar delim = ls->c;  /* Delimiter is '\'' or '"'. */
 185   lex_savenext(ls);
 186   while (ls->c != delim) {
 187     switch (ls->c) {
 188     case LEX_EOF:
 189       lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
 190       continue;
 191     case '\n':
 192     case '\r':
 193       lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
 194       continue;
 195     case '\\': {
 196       LexChar c = lex_next(ls);  /* Skip the '\\'. */
 197       switch (c) {
 198       case 'a': c = '\a'; break;
 199       case 'b': c = '\b'; break;
 200       case 'f': c = '\f'; break;
 201       case 'n': c = '\n'; break;
 202       case 'r': c = '\r'; break;
 203       case 't': c = '\t'; break;
 204       case 'v': c = '\v'; break;
 205       case 'x':  /* Hexadecimal escape '\xXX'. */
 206         c = (lex_next(ls) & 15u) << 4;
 207         if (!lj_char_isdigit(ls->c)) {
 208           if (!lj_char_isxdigit(ls->c)) goto err_xesc;
 209           c += 9 << 4;
 210         }
 211         c += (lex_next(ls) & 15u);
 212         if (!lj_char_isdigit(ls->c)) {
 213           if (!lj_char_isxdigit(ls->c)) goto err_xesc;
 214           c += 9;
 215         }
 216         break;
 217       case 'u':  /* Unicode escape '\u{XX...}'. */
 218         if (lex_next(ls) != '{') goto err_xesc;
 219         lex_next(ls);
 220         c = 0;
 221         do {
 222           c = (c << 4) | (ls->c & 15u);
 223           if (!lj_char_isdigit(ls->c)) {
 224             if (!lj_char_isxdigit(ls->c)) goto err_xesc;
 225             c += 9;
 226           }
 227           if (c >= 0x110000) goto err_xesc;  /* Out of Unicode range. */
 228         } while (lex_next(ls) != '}');
 229         if (c < 0x800) {
 230           if (c < 0x80) break;
 231           lex_save(ls, 0xc0 | (c >> 6));
 232         } else {
 233           if (c >= 0x10000) {
 234             lex_save(ls, 0xf0 | (c >> 18));
 235             lex_save(ls, 0x80 | ((c >> 12) & 0x3f));
 236           } else {
 237             if (c >= 0xd800 && c < 0xe000) goto err_xesc;  /* No surrogates. */
 238             lex_save(ls, 0xe0 | (c >> 12));
 239           }
 240           lex_save(ls, 0x80 | ((c >> 6) & 0x3f));
 241         }
 242         c = 0x80 | (c & 0x3f);
 243         break;
 244       case 'z':  /* Skip whitespace. */
 245         lex_next(ls);
 246         while (lj_char_isspace(ls->c))
 247           if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls);
 248         continue;
 249       case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue;
 250       case '\\': case '\"': case '\'': break;
 251       case LEX_EOF: continue;
 252       default:
 253         if (!lj_char_isdigit(c))
 254           goto err_xesc;
 255         c -= '0';  /* Decimal escape '\ddd'. */
 256         if (lj_char_isdigit(lex_next(ls))) {
 257           c = c*10 + (ls->c - '0');
 258           if (lj_char_isdigit(lex_next(ls))) {
 259             c = c*10 + (ls->c - '0');
 260             if (c > 255) {
 261             err_xesc:
 262               lj_lex_error(ls, TK_string, LJ_ERR_XESC);
 263             }
 264             lex_next(ls);
 265           }
 266         }
 267         lex_save(ls, c);
 268         continue;
 269       }
 270       lex_save(ls, c);
 271       lex_next(ls);
 272       continue;
 273       }
 274     default:
 275       lex_savenext(ls);
 276       break;
 277     }
 278   }
 279   lex_savenext(ls);  /* Skip trailing delimiter. */
 280   setstrV(ls->L, tv,
 281           lj_parse_keepstr(ls, sbufB(&ls->sb)+1, sbuflen(&ls->sb)-2));
 282 }
 283 
 284 /* -- Main lexical scanner ------------------------------------------------ */
 285 
 286 /* Get next lexical token. */
 287 static LexToken lex_scan(LexState *ls, TValue *tv)
 288 {
 289   lj_buf_reset(&ls->sb);
 290   for (;;) {
 291     if (lj_char_isident(ls->c)) {
 292       GCstr *s;
 293       if (lj_char_isdigit(ls->c)) {  /* Numeric literal. */
 294         lex_number(ls, tv);
 295         return TK_number;
 296       }
 297       /* Identifier or reserved word. */
 298       do {
 299         lex_savenext(ls);
 300       } while (lj_char_isident(ls->c));
 301       s = lj_parse_keepstr(ls, sbufB(&ls->sb), sbuflen(&ls->sb));
 302       setstrV(ls->L, tv, s);
 303       if (s->reserved > 0)  /* Reserved word? */
 304         return TK_OFS + s->reserved;
 305       return TK_name;
 306     }
 307     switch (ls->c) {
 308     case '\n':
 309     case '\r':
 310       lex_newline(ls);
 311       continue;
 312     case ' ':
 313     case '\t':
 314     case '\v':
 315     case '\f':
 316       lex_next(ls);
 317       continue;
 318     case '-':
 319       lex_next(ls);
 320       if (ls->c != '-') return '-';
 321       lex_next(ls);
 322       if (ls->c == '[') {  /* Long comment "--[=*[...]=*]". */
 323         int sep = lex_skipeq(ls);
 324         lj_buf_reset(&ls->sb);  /* `lex_skipeq' may dirty the buffer */
 325         if (sep >= 0) {
 326           lex_longstring(ls, NULL, sep);
 327           lj_buf_reset(&ls->sb);
 328           continue;
 329         }
 330       }
 331       /* Short comment "--.*\n". */
 332       while (!lex_iseol(ls) && ls->c != LEX_EOF)
 333         lex_next(ls);
 334       continue;
 335     case '[': {
 336       int sep = lex_skipeq(ls);
 337       if (sep >= 0) {
 338         lex_longstring(ls, tv, sep);
 339         return TK_string;
 340       } else if (sep == -1) {
 341         return '[';
 342       } else {
 343         lj_lex_error(ls, TK_string, LJ_ERR_XLDELIM);
 344         continue;
 345       }
 346       }
 347     case '=':
 348       lex_next(ls);
 349       if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; }
 350     case '<':
 351       lex_next(ls);
 352       if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; }
 353     case '>':
 354       lex_next(ls);
 355       if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; }
 356     case '~':
 357       lex_next(ls);
 358       if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; }
 359     case ':':
 360       lex_next(ls);
 361       if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; }
 362     case '"':
 363     case '\'':
 364       lex_string(ls, tv);
 365       return TK_string;
 366     case '.':
 367       if (lex_savenext(ls) == '.') {
 368         lex_next(ls);
 369         if (ls->c == '.') {
 370           lex_next(ls);
 371           return TK_dots;   /* ... */
 372         }
 373         return TK_concat;   /* .. */
 374       } else if (!lj_char_isdigit(ls->c)) {
 375         return '.';
 376       } else {
 377         lex_number(ls, tv);
 378         return TK_number;
 379       }
 380     case LEX_EOF:
 381       return TK_eof;
 382     default: {
 383       LexChar c = ls->c;
 384       lex_next(ls);
 385       return c;  /* Single-char tokens (+ - / ...). */
 386     }
 387     }
 388   }
 389 }
 390 
 391 /* -- Lexer API ----------------------------------------------------------- */
 392 
 393 /* Setup lexer state. */
 394 int lj_lex_setup(lua_State *L, LexState *ls)
 395 {
 396   int header = 0;
 397   ls->L = L;
 398   ls->fs = NULL;
 399   ls->pe = ls->p = NULL;
 400   ls->vstack = NULL;
 401   ls->sizevstack = 0;
 402   ls->vtop = 0;
 403   ls->bcstack = NULL;
 404   ls->sizebcstack = 0;
 405   ls->tok = 0;
 406   ls->lookahead = TK_eof;  /* No look-ahead token. */
 407   ls->linenumber = 1;
 408   ls->lastline = 1;
 409   lex_next(ls);  /* Read-ahead first char. */
 410   if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb &&
 411       (uint8_t)ls->p[1] == 0xbf) {  /* Skip UTF-8 BOM (if buffered). */
 412     ls->p += 2;
 413     lex_next(ls);
 414     header = 1;
 415   }
 416   if (ls->c == '#') {  /* Skip POSIX #! header line. */
 417     do {
 418       lex_next(ls);
 419       if (ls->c == LEX_EOF) return 0;
 420     } while (!lex_iseol(ls));
 421     lex_newline(ls);
 422     header = 1;
 423   }
 424   if (ls->c == LUA_SIGNATURE[0]) {  /* Bytecode dump. */
 425     if (header) {
 426       /*
 427       ** Loading bytecode with an extra header is disabled for security
 428       ** reasons. This may circumvent the usual check for bytecode vs.
 429       ** Lua code by looking at the first char. Since this is a potential
 430       ** security violation no attempt is made to echo the chunkname either.
 431       */
 432       setstrV(L, L->top++, lj_err_str(L, LJ_ERR_BCBAD));
 433       lj_err_throw(L, LUA_ERRSYNTAX);
 434     }
 435     return 1;
 436   }
 437   return 0;
 438 }
 439 
 440 /* Cleanup lexer state. */
 441 void lj_lex_cleanup(lua_State *L, LexState *ls)
 442 {
 443   global_State *g = G(L);
 444   lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
 445   lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
 446   lj_buf_free(g, &ls->sb);
 447 }
 448 
 449 /* Return next lexical token. */
 450 void lj_lex_next(LexState *ls)
 451 {
 452   ls->lastline = ls->linenumber;
 453   if (LJ_LIKELY(ls->lookahead == TK_eof)) {  /* No lookahead token? */
 454     ls->tok = lex_scan(ls, &ls->tokval);  /* Get next token. */
 455   } else {  /* Otherwise return lookahead token. */
 456     ls->tok = ls->lookahead;
 457     ls->lookahead = TK_eof;
 458     ls->tokval = ls->lookaheadval;
 459   }
 460 }
 461 
 462 /* Look ahead for the next token. */
 463 LexToken lj_lex_lookahead(LexState *ls)
 464 {
 465   lua_assert(ls->lookahead == TK_eof);
 466   ls->lookahead = lex_scan(ls, &ls->lookaheadval);
 467   return ls->lookahead;
 468 }
 469 
 470 /* Convert token to string. */
 471 const char *lj_lex_token2str(LexState *ls, LexToken tok)
 472 {
 473   if (tok > TK_OFS)
 474     return tokennames[tok-TK_OFS-1];
 475   else if (!lj_char_iscntrl(tok))
 476     return lj_strfmt_pushf(ls->L, "%c", tok);
 477   else
 478     return lj_strfmt_pushf(ls->L, "char(%d)", tok);
 479 }
 480 
 481 /* Lexer error. */
 482 void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...)
 483 {
 484   const char *tokstr;
 485   va_list argp;
 486   if (tok == 0) {
 487     tokstr = NULL;
 488   } else if (tok == TK_name || tok == TK_string || tok == TK_number) {
 489     lex_save(ls, '\0');
 490     tokstr = sbufB(&ls->sb);
 491   } else {
 492     tokstr = lj_lex_token2str(ls, tok);
 493   }
 494   va_start(argp, em);
 495   lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp);
 496   va_end(argp);
 497 }
 498 
 499 /* Initialize strings for reserved words. */
 500 void lj_lex_init(lua_State *L)
 501 {
 502   uint32_t i;
 503   for (i = 0; i < TK_RESERVED; i++) {
 504     GCstr *s = lj_str_newz(L, tokennames[i]);
 505     fixstring(s);  /* Reserved words are never collected. */
 506     s->reserved = (uint8_t)(i+1);
 507   }
 508 }
 509 

/* [<][>][^][v][top][bottom][index][help] */