home *** CD-ROM | disk | FTP | other *** search
/ Usenet 1994 October / usenetsourcesnewsgroupsinfomagicoctober1994disk2.iso / net / c++.scanner < prev    next >
Internet Message Format  |  1987-01-16  |  32KB
  1. From nmyers@mntgfx.MENTOR.COM Wed Jan 14 17:35:26 1987
  2. Path: beno!seismo!husc6!panda!genrad!decvax!tektronix!sequent!mntgfx!nmyers
  3. From: nmyers@mntgfx.MENTOR.COM (Nathan Myers)
  4. Newsgroups: net.sources
  5. Subject: C++ Lexical Scanner (in C++)
  6. Keywords: object parse compiler cfront
  7. Message-ID: <431@mntgfx.MENTOR.COM>
  8. Date: 14 Jan 87 22:35:26 GMT
  9. Organization: Mentor Graphics, Beaverton OR
  10. Lines: 1054
  11.  
  12. C++ Lexical Scanner (in C++)
  13.  
  14. Following this note is a shar-format file containing a
  15. lexical scanner for C++.  Use it however you like.  It
  16. should be adequate to drop into a full compiler.
  17. For such a use, error reporting would probably need 
  18. some refinement.  Also, if the compiler was to have
  19. a built-in preprocessor (i.e. not using cpp) the handler
  20. for "#" lines would need a bit of work.
  21.  
  22. I will be interested in bug reports, significant improvements.
  23. Nathan Myers   tektronix!sequent!mntgfx!nmyers
  24.  
  25. Share and enjoy!
  26. -------------------- cut here -----------------------------
  27. #! /bin/sh
  28. #  This is a shar format file.  Extract with sh, not csh.
  29. #
  30. echo "x - read.me"
  31. sed -e 's/^_ //' >read.me <<'%%%EOF%%%'
  32. _ 
  33. _ The following source files are included:
  34. _ 
  35. _     clex.c
  36. _     clex.h
  37. _     clex_sym.h
  38. _     clex_test.c
  39. _     kwhash.c
  40. _     Makefile
  41. _ 
  42. _ They implement a self-contained lexical scanner class for C++.
  43. _ It is extensible by derivation primarily in the area of
  44. _ processing "#" compiler directives (currently, it only
  45. _ interprets the "#line" construct).
  46. _ 
  47. _ It has one other degree of flexibility, in its handling
  48. _ of bracket-enclosed expressions "[]".  These may be treated
  49. _ as a normal sequence of tokens or as delimited strings;
  50. _ the former is of greater use in a traditional parser, while
  51. _ the latter is favored for extraction of declarations by
  52. _ a code browser.
  53. _ 
  54. _ To allay some confusion, I should point out here that
  55. _ clex_sym.h is used in an unusual way: it is included
  56. _ twice in the module clex.c; once for declaration
  57. _ part, once for the (static) definition part.  It is
  58. _ built this way to keep all knowledge of keywords
  59. _ in a single place.
  60. _ 
  61. _ The file kwhash.c is a standard C standalone program used
  62. _ to arrive at the collision-free hash function used to
  63. _ recognize C++ keywords.  Any new keyword stands about
  64. _ one chance in 3 of colliding with an existing keyword,
  65. _ thus requiring that a new hash function be generated.
  66. _ 
  67. _ The file clex_test.c compiles to a program which reads
  68. _ C or C++ code from standard input and emits token names
  69. _ on the standard output.  Try it with different values
  70. _ in the constructor's second argument.
  71. %%%EOF%%%
  72. echo "x - Makefile"
  73. sed -e 's/^_ //' >Makefile <<'%%%EOF%%%'
  74. _ # Makefile for clex/cparse
  75. _ INCLUDE=/usr/local/include/CC
  76. _ CCC=/user/local/CC
  77. _ CCOPTS= -g -O
  78. _ # -g: include debug info  -O: optimize
  79. _ 
  80. _ all: kwhash clex_test
  81. _ 
  82. _ kwhash: kwhash.c /usr/include/stdio.h
  83. _         /bin/cc ${CCOPTS} -o kwhash kwhash.c
  84. _ 
  85. _ clex_test: clex_test.o clex.o
  86. _         ${CCC} -g -o clex_test clex_test.o clex.o
  87. _ 
  88. _ .c.o:
  89. _         /usr/lib/cpp -I${INCLUDE} -Dc_plusplus=1 $*.c >$*.cpp
  90. _         /user/mentor/local/cfront +L +f$*.c <$*.cpp >$*..c && /bin/rm $*.cpp
  91. _         /bin/cc -c ${CCOPTS} $*..c && /bin/mv $*..o $*.o
  92. _ 
  93. _ clex.o: ${INCLUDE}/stdio.h \
  94. _          ${INCLUDE}/stream.h \
  95. _          ${INCLUDE}/string.h \
  96. _          ${INCLUDE}/stdlib.h \
  97. _          ${INCLUDE}/ctype.h \
  98. _          ${INCLUDE}/assert.h \
  99. _          clex.h clex_sym.h
  100. _ 
  101. _ clex_test.o:    ${INCLUDE}/stdio.h \
  102. _                 clex.h clex_sym.h
  103. _ 
  104. %%%EOF%%%
  105. echo "x - clex.h"
  106. sed -e 's/^_ //' >clex.h <<'%%%EOF%%%'
  107. _ 
  108. _ 
  109. _ #ifndef INCLUDED_CLEX
  110. _ #define INCLUDED_CLEX 1
  111. _ 
  112. _ #ifndef INCLUDED_STDIO
  113. _ #include <stdio.h>
  114. _ #endif
  115. _ 
  116. _ enum Boolean { FALSE, TRUE };
  117. _ 
  118. _ #include "clex_sym.h"
  119. _ 
  120. _ class Clex
  121. _     { 
  122. _  friend class Cparse;
  123. _     enum Clex_mode
  124. _         { CL_NONE=0, CL_COMMENT=1, CL_QUOTE=2, CL_POUND=4, CL_BRACK=8 };
  125. _ 
  126. _  protected:
  127. _     short   look;           // a one-char lookahead
  128. _     FILE*   fp;
  129. _     Boolean block_brack;    // if TRUE, treat contents of "[]" as a string
  130. _     long    line_num;       // line number in original source file
  131. _     char    filename[256];  // name of original source file
  132. _     short   bufsiz;         // number of chars currently in buf
  133. _     char    buf[256];
  134. _ 
  135. _     void eat_one()          { look = short(getc(fp)); }
  136. _     void put_in_buf(char c) { if (bufsiz < sizeof(buf)-1) buf[bufsiz++] = c; }
  137. _     void buf_one()          { put_in_buf(look); eat_one(); }
  138. _     Clex_sym terminate(Clex_sym s)  { buf[bufsiz] = '\0'; return s; }
  139. _     Clex_sym eat_return(Clex_sym);
  140. _     Clex_sym num(char);
  141. _     Clex_sym ident(char);
  142. _     Clex_sym lbrack(Clex_mode);
  143. _     Clex_sym quote(char, Clex_sym, Clex_mode);
  144. _     void block_comment(Clex_mode);
  145. _     void line_comment();
  146. _     void eoln(Clex_mode);
  147. _ 
  148. _     virtual Boolean pound(Clex_mode, char*, short len);
  149. _ 
  150. _  public:
  151. _     Clex_sym    next();
  152. _     const char* str()           { return buf; }
  153. _     short       strlen()        { return bufsiz; }
  154. _     long        line_no()       { return line_num; }
  155. _     const char* fname()         { return filename; }
  156. _     const char* debug(Clex_sym);
  157. _ 
  158. _     Clex(FILE*, Boolean block_brack);
  159. _     };
  160. _ 
  161. _ #endif
  162. _ 
  163. %%%EOF%%%
  164. echo "x - clex_sym.h"
  165. sed -e 's/^_ //' >clex_sym.h <<'%%%EOF%%%'
  166. _ 
  167. _ /* clex_sym.h:
  168. _ // This file defines both an enum {} type named "symbol", and
  169. _ //  a variable sym_str[] with string representations of the
  170. _ //  symbols.  It is intended to maintain an exact
  171. _ //  correspondence between array entries and symbol values.
  172. _ */
  173. _ 
  174. _ /*
  175. _     This file is #include'd twice: once for the enum
  176. _     (with CLEX_IMPLEMENTATION turned off) and once for
  177. _     the array initialization (with it turned on).  The
  178. _     lower-numbered symbols have uppercase name strings,
  179. _     but the keyword symbol strings are stored separately.
  180. _ 
  181. _     If a keyword is to be added, add it first to the
  182. _     standalone program kwhash.c and generate a new
  183. _     perfect hash function for the new set.  Then add
  184. _     it to both places below and modify the hash function
  185. _     and table size in clex.c.
  186. _ */
  187. _ 
  188. _ #ifndef CLEX_IMPLEMENTATION
  189. _ 
  190. _ #define CLEX_S(sym) sym
  191. _ #define CLEX_S2(sym1,sym2) sym1
  192. _ enum Clex_sym
  193. _     {
  194. _ 
  195. _ #else /* CLEX_IMPLEMENTATION */
  196. _ 
  197. _ #undef  CLEX_S
  198. _ #undef  CLEX_S2
  199. _ #define CLEX_S(sym) "sym"
  200. _ #define CLEX_S2(sym1,sym2) sym2
  201. _ static char* sym_str[] =
  202. _     {
  203. _ 
  204. _ #endif /* CLEX_IMPLEMENTATION */
  205. _ 
  206. _     CLEX_S(NONE_S = 0),    /* should never get this */
  207. _ 
  208. _     CLEX_S(ERROR_S),
  209. _     CLEX_S(  ERROR_EOLN_S),
  210. _     CLEX_S(  ERROR_EOF_S),
  211. _     CLEX_S(  ERROR_UNKN_S),
  212. _ #ifndef CLEX_IMPLEMENTATION
  213. _     CLEX_S(ERROR_MAX = ERROR_UNKN_S),
  214. _ #endif
  215. _ 
  216. _     CLEX_S(EOF_S),        
  217. _     CLEX_S(EOLN_S),         // \n
  218. _ 
  219. _     CLEX_S(BANG_S),         // !
  220. _     CLEX_S(  NE_S),         // !=
  221. _     CLEX_S(QUOTE_S),        // "
  222. _     CLEX_S(POUND_S),        // #
  223. _     CLEX_S(MOD_S),          // %
  224. _     CLEX_S(  MODAS_S),      // %=
  225. _     CLEX_S(AMPER_S),        // &
  226. _     CLEX_S(  LAND_S),       // &&
  227. _     CLEX_S(  ANDAS_S),      // &=
  228. _     CLEX_S(APOS_S),         // '
  229. _     CLEX_S(LPAR_S),         // (
  230. _     CLEX_S(RPAR_S),         // )
  231. _     CLEX_S(STAR_S),         // *
  232. _     CLEX_S(  MULAS_S),      // *=
  233. _     CLEX_S(PLUS_S),         // +
  234. _     CLEX_S(  INCRE_S),      // ++
  235. _     CLEX_S(  ADDAS_S),      // +=
  236. _     CLEX_S(COMMA_S),        // ),
  237. _     CLEX_S(MINUS_S),        // -
  238. _     CLEX_S(  DECRE_S),      // --
  239. _     CLEX_S(  SUBAS_S),      // -=
  240. _     CLEX_S(  DEREF_S),      // ->
  241. _     CLEX_S(DOT_S),          // .
  242. _     CLEX_S(  ELLIP_S),      // ...
  243. _     CLEX_S(SLASH_S),        // / 
  244. _     CLEX_S(  DIVAS_S),      // /=
  245. _     CLEX_S(COLON_S),        // :
  246. _     CLEX_S(  SCOPE_S),      // ::
  247. _     CLEX_S(SEMI_S),         // ;
  248. _     CLEX_S(LT_S),           // <
  249. _     CLEX_S(  LE_S),         // <=
  250. _     CLEX_S(  SHL_S),        // <<
  251. _     CLEX_S(  SHLAS_S),      // <<=
  252. _     CLEX_S(AS_S),           // =
  253. _     CLEX_S(  EQ_S),         // ==
  254. _     CLEX_S(GT_S),           // >
  255. _     CLEX_S(  GE_S),         // >=
  256. _     CLEX_S(  SHR_S),        // >>
  257. _     CLEX_S(  SHRAS_S),      // >>=
  258. _     CLEX_S(QUEST_S),        // ?
  259. _     CLEX_S(AT_S),           // @ (undefined)
  260. _     CLEX_S(LBRACK_S),       // [
  261. _     CLEX_S(BSLASH_S),       // \ 
  262. _     CLEX_S(RBRACK_S),       // ]
  263. _     CLEX_S(CARET_S),        // ^
  264. _     CLEX_S(  XORAS_S),      // ^=
  265. _     CLEX_S(GRAVE_S),        // ` (undefined)
  266. _     CLEX_S(LBRACE_S),       // {
  267. _     CLEX_S(VBAR_S),         // |
  268. _     CLEX_S(  LOR_S),        // ||
  269. _     CLEX_S(  ORAS_S),       // |=
  270. _     CLEX_S(RBRACE_S),       // }
  271. _     CLEX_S(TILDE_S),        // ~
  272. _ 
  273. _     CLEX_S(IDENT_S),        // a name, or string that could be a name
  274. _     CLEX_S(NUM_S),          // a numeric string
  275. _     CLEX_S(FLOATNUM_S)      // a recognizably floating-point num
  276. _ 
  277. _ #ifndef CLEX_IMPLEMENTATION
  278. _     , CLEX_S(KEYWORD_S),
  279. _ 
  280. _ #else
  281. _ 
  282. _     };
  283. _  static char *keywords[] =
  284. _     {
  285. _ 
  286. _ #endif
  287. _ 
  288. _     CLEX_S2(ASM_S = KEYWORD_S, "asm"),
  289. _     CLEX_S2(AUTO_S,     "auto"),
  290. _     CLEX_S2(BREAK_S,    "break"),
  291. _     CLEX_S2(CASE_S,     "case"),
  292. _     CLEX_S2(CHAR_S,     "char"),
  293. _     CLEX_S2(CLASS_S,    "class"),
  294. _     CLEX_S2(CONST_S,    "const"),
  295. _     CLEX_S2(CONTINUE_S, "continue"),
  296. _     CLEX_S2(DEFAULT_S,  "default"),
  297. _     CLEX_S2(DELETE_S,   "delete"),
  298. _     CLEX_S2(DO_S,       "do"),
  299. _     CLEX_S2(DOUBLE_S,   "double"),
  300. _     CLEX_S2(ELSE_S,     "else"),
  301. _     CLEX_S2(ENUM_S,     "enum"),
  302. _     CLEX_S2(EXTERN_S,   "extern"),
  303. _     CLEX_S2(FLOAT_S,    "float"),
  304. _     CLEX_S2(FOR_S,      "for"),
  305. _     CLEX_S2(FRIEND_S,   "friend"),
  306. _     CLEX_S2(GOTO_S,     "goto"),
  307. _     CLEX_S2(IF_S,       "if"),
  308. _     CLEX_S2(INLINE_S,   "inline"),
  309. _     CLEX_S2(INT_S,      "int"),
  310. _     CLEX_S2(LONG_S,     "long"),
  311. _     CLEX_S2(NEW_S,      "new"),
  312. _     CLEX_S2(OPERATOR_S, "operator"),
  313. _     CLEX_S2(OVERLOAD_S, "overload"),
  314. _     CLEX_S2(PRIVATE_S,  "private"),
  315. _     CLEX_S2(PROTECTED_S,"protected"),
  316. _     CLEX_S2(PUBLIC_S,   "public"),
  317. _     CLEX_S2(REGISTER_S, "register"),
  318. _     CLEX_S2(RETURN_S,   "return"),
  319. _     CLEX_S2(SHORT_S,    "short"),
  320. _     CLEX_S2(SIGNED_S,   "signed"),
  321. _     CLEX_S2(SIZEOF_S,   "sizeof"),
  322. _     CLEX_S2(STATIC_S,   "static"),
  323. _     CLEX_S2(STRUCT_S,   "struct"),
  324. _     CLEX_S2(SWITCH_S,   "switch"),
  325. _     CLEX_S2(THIS_S,     "this"),
  326. _     CLEX_S2(TYPEDEF_S,  "typedef"),
  327. _     CLEX_S2(UNION_S,    "union"),
  328. _     CLEX_S2(UNSIGNED_S, "unsigned"),
  329. _     CLEX_S2(VIRTUAL_S,  "virtual"),
  330. _     CLEX_S2(VOLATILE_S, "volatile"),
  331. _     CLEX_S2(VOID_S,     "void"),
  332. _     CLEX_S2(WHILE_S,    "while"),
  333. _ 
  334. _     CLEX_S2(END_OF_SYMBOLS_S, NULL)
  335. _     };
  336. _ 
  337. _ #ifndef CLEX_IMPLEMENTATION
  338. _ const CLEX_NUMKEYS = (END_OF_SYMBOLS_S - KEYWORD_S);
  339. _ #endif
  340. _ 
  341. %%%EOF%%%
  342. echo "x - clex.c"
  343. sed -e 's/^_ //' >clex.c <<'%%%EOF%%%'
  344. _ 
  345. _ #ifndef INCLUDED_STREAM
  346. _ #include <stream.h>
  347. _ #endif
  348. _ #ifndef INCLUDED_STRING
  349. _ #include <string.h>
  350. _ #endif
  351. _ #ifndef INCLUDED_STDLIB
  352. _ #include <stdlib.h>
  353. _ #endif
  354. _ #ifndef INCLUDED_ASSERT
  355. _ #include <assert.h>
  356. _ #endif
  357. _ #ifndef INCLUDED_CTYPE
  358. _ #include <ctype.h>
  359. _ #endif
  360. _ 
  361. _ #include "clex.h"
  362. _ 
  363. _ // get string value tables, sym_str[] and keyword[] :
  364. _ #define CLEX_IMPLEMENTATION 1
  365. _ #include "clex_sym.h"
  366. _ 
  367. _ /******************************************************************************
  368. _ *                                                                             *
  369. _ *  KWTABLE -- keyword hash table (internal use only)                          *
  370. _ *     KWtable implements a collision-free hash table of C++ keywords.  The    *
  371. _ *     table size and hash function are computed by use of a standalone C      *
  372. _ *     program, kwhash.c, included in this directory.                          *
  373. _ *                                                                             *
  374. _ ******************************************************************************/
  375. _ 
  376. _ #define U_short unsigned short
  377. _ #define U_char  unsigned char
  378. _ 
  379. _ struct KWtable
  380. _     {
  381. _     enum { HASHSIZE = 131 };  // as computed by kwhash.c, for a=9,b=2,c=2
  382. _ 
  383. _     struct  {
  384. _             char* kwp;
  385. _             Clex_sym sym;
  386. _             } kwhash[HASHSIZE];
  387. _ 
  388. _     KWtable(char**);
  389. _     U_short hash(const U_char*, U_short len);
  390. _     void insert(char*, Clex_sym);
  391. _     Clex_sym lookup(char*, short len);
  392. _     };
  393. _ 
  394. _ static KWtable kwt = KWtable(keywords); // keywords[] defined in Clex_sym.h
  395. _ 
  396. _ KWtable::
  397. _ KWtable (char** kwl)
  398. _     {
  399. _     short int i;
  400. _     for (i = 0; i < HASHSIZE; ++i)
  401. _         kwhash[i].kwp = NULL;
  402. _     for (i = 0; i < CLEX_NUMKEYS; ++i)
  403. _         insert(kwl[i], KEYWORD_S + i);
  404. _     // rely on assert() to prevent hash collisions -- may need
  405. _     //  a new hash function or table size when keyword added.
  406. _     }
  407. _ 
  408. _ // the values used in the following hash function, and HASHSIZE, were
  409. _ // determined by use of the standalone C program kwhash.c, to
  410. _ // ensure that no collisions occur.
  411. _ 
  412. _ inline
  413. _ U_short KWtable::
  414. _ hash (const U_char* cp, U_short len)
  415. _     {
  416. _     return (((U_short)cp[0]         ) ^
  417. _             ((U_short)cp[1]     << 9) ^
  418. _             ((U_short)cp[len-1] << 2) ^
  419. _             (len                << 2) ) % HASHSIZE;
  420. _     }
  421. _ 
  422. _ void KWtable::
  423. _ insert (char* cp, Clex_sym s)
  424. _     {
  425. _     U_short h = hash(cp, strlen(cp));
  426. _     assert(kwt.kwhash[h].kwp == NULL);  // collisions not permitted.
  427. _     kwt.kwhash[h].kwp = cp;
  428. _     kwt.kwhash[h].sym = s;
  429. _     }
  430. _ 
  431. _ Clex_sym KWtable::
  432. _ lookup (char* cp, short len)
  433. _     {
  434. _     if (len < 2 || len > 9) return (IDENT_S);
  435. _     short h = hash(cp, len);
  436. _     if (kwt.kwhash[h].kwp == NULL) return (IDENT_S);
  437. _     if (strcmp(kwt.kwhash[h].kwp, cp)) return (IDENT_S);
  438. _     return (kwt.kwhash[h].sym);
  439. _     }
  440. _ 
  441. _ /******************************************************************************
  442. _ *                                                                             *
  443. _ *  CLEX -- c++ lexical scanner                                               *
  444. _ *                                                                             *
  445. _ ******************************************************************************/
  446. _ 
  447. _ // CONSTRUCTOR Clex:
  448. _ //   The argument block_brack, if TRUE, dictates that the contents
  449. _ //   of square brackets "[]" be returned as a string in the string
  450. _ //   buffer.  If false, square brackets are treated as simple tokens.
  451. _ 
  452. _ Clex::
  453. _ Clex (FILE* f, Boolean b)
  454. _     {
  455. _     fp = f;
  456. _     block_brack = b;
  457. _     filename[0] = '\0';
  458. _     bufsiz = 0; buf[0] = '\0';
  459. _     // prime the pipeline:
  460. _     line_num = 0;
  461. _     look = '\n';    // be prepared to handle '#' as first char
  462. _     }
  463. _ 
  464. _ Clex_sym Clex::
  465. _ num (char c)
  466. _     {
  467. _     Clex_sym s = NUM_S;
  468. _ 
  469. _     bufsiz = 0;
  470. _     put_in_buf(c);
  471. _     while (isdigit(look))
  472. _         buf_one();
  473. _ 
  474. _     // hexadecimal
  475. _     if (bufsiz == 1 && *buf == '0' && (look == 'x' || look == 'X'))
  476. _         {
  477. _         do { buf_one(); }
  478. _             while (isxdigit(look));
  479. _         if (look == 'L' || look == 'l' || look == 'U' || look == 'u')
  480. _             buf_one();
  481. _         return terminate(s);
  482. _         }
  483. _ 
  484. _     // long or unsigned
  485. _     if (look == 'L' || look == 'l' || look == 'U' || look == 'u')
  486. _         { buf_one(); return terminate(NUM_S); }
  487. _ 
  488. _     // floating point
  489. _     else if (look == '.')
  490. _         {
  491. _         s = FLOATNUM_S;
  492. _         do { buf_one(); }
  493. _             while (isdigit(look));
  494. _         }
  495. _ 
  496. _     // scientific notation
  497. _     if (look == 'e' || look == 'E')
  498. _          {
  499. _          s = FLOATNUM_S;
  500. _          do { buf_one(); }
  501. _             while (isdigit(look));
  502. _          }
  503. _     else
  504. _         return terminate(s);
  505. _ 
  506. _     if (look == '+' || look == '-')
  507. _          do { buf_one(); }
  508. _             while (isdigit(look));
  509. _     return terminate(s);
  510. _     }
  511. _ 
  512. _ Clex_sym Clex::
  513. _ ident (char first)
  514. _     {
  515. _     register Boolean maybe_kw = TRUE;
  516. _     register short bs = 0;
  517. _     buf[bs++] = first;
  518. _     while (isalnum(look) || look == '_' || look == '$')
  519. _         {
  520. _         // note: this function accounts for 30% of the total scan time
  521. _         if (maybe_kw && (isupper(look) || look == '_' ))
  522. _             maybe_kw = FALSE;
  523. _         buf[bs++] = look;       // don't worry about overflow
  524. _         eat_one();
  525. _         }
  526. _     buf[bs] = '\0';
  527. _     bufsiz = bs;
  528. _ 
  529. _     if (maybe_kw)
  530. _         return kwt.lookup(buf, bufsiz);
  531. _     return IDENT_S;
  532. _     }
  533. _ 
  534. _ Clex_sym Clex::
  535. _ quote (char c, Clex_sym s, Clex_mode m)
  536. _     {
  537. _     if (m == CL_NONE)
  538. _         bufsiz = 0;
  539. _     while (look != c)
  540. _         {
  541. _         if (look == EOF)
  542. _             { return terminate(ERROR_EOF_S); }
  543. _         else if (look == '\n')
  544. _             { return terminate(ERROR_EOLN_S); }
  545. _         else if (look == '\\')
  546. _             {
  547. _             eat_one();
  548. _             if (look == '\n')
  549. _                 { eat_one(); eoln(m|CL_QUOTE); continue; }
  550. _             else if (look == EOF)
  551. _                 { return terminate(ERROR_EOF_S); }
  552. _             else
  553. _                 put_in_buf('\\');   // this handles \' and \" too.
  554. _             }
  555. _         buf_one();
  556. _         }
  557. _     eat_one();  // eat the closing quote
  558. _     return terminate(s);
  559. _     }
  560. _ 
  561. _ 
  562. _ // lbrack() accumulates the contents between "[" and "]" into
  563. _ //  the string buffer, handling syntactically quoted strings,
  564. _ //  comments, and nested brackets.  Note that lbrack() is
  565. _ //  called recursively in the case of nested brackets.
  566. _ 
  567. _ Clex_sym Clex::
  568. _ lbrack (Clex_mode m)
  569. _     {
  570. _     if (m == CL_NONE)
  571. _         bufsiz = 0;
  572. _     while (look != ']')
  573. _         {
  574. _         if (look == EOF)
  575. _             return terminate(ERROR_EOF_S);
  576. _ 
  577. _         else if (look == '\n')
  578. _             { eat_one(); eoln(m|CL_BRACK); }
  579. _         else if (look == '[')
  580. _             {
  581. _             buf_one();
  582. _             if (lbrack(m|CL_BRACK) == ERROR_EOF_S)
  583. _                 return ERROR_EOF_S;     // already cleaned up.
  584. _             else put_in_buf(']');
  585. _             }
  586. _         else if (look == '\'' || look == '"')
  587. _             {
  588. _             char c = look;
  589. _             buf_one();
  590. _             (void) quote(c, NONE_S, m|CL_BRACK);
  591. _             put_in_buf(c);
  592. _             }
  593. _         else if (look == '/')           // maybe a comment
  594. _             {
  595. _             eat_one();
  596. _             if (look == '/')
  597. _                 line_comment();
  598. _             else if (look == '*')
  599. _                 {
  600. _                 block_comment(m|CL_BRACK);
  601. _                 if (look == EOF) return terminate(ERROR_EOF_S);
  602. _                 }
  603. _             else                        // stash the '/' and the char after
  604. _                 { put_in_buf('/'); buf_one(); }
  605. _             }
  606. _         else                            // just a character to save
  607. _             buf_one();
  608. _         }
  609. _ 
  610. _     eat_one(); // eat the ']'.
  611. _     return terminate(LBRACK_S);
  612. _     }
  613. _ 
  614. _ 
  615. _ void Clex::
  616. _ block_comment(Clex_mode m)
  617. _     {
  618. _     eat_one(); // eat the '*'
  619. _     while (! (look == '*' && (eat_one(), look == '/')) )
  620. _         {
  621. _         if (look == EOF) return;
  622. _         if (look == '\n') { eat_one(); eoln(m|CL_COMMENT); }
  623. _         else if (look != '*') eat_one();
  624. _         }
  625. _     eat_one(); // eat the '/'
  626. _     }
  627. _ 
  628. _ void Clex::
  629. _ line_comment()
  630. _     {
  631. _     do { eat_one(); }
  632. _      while (look != '\n' && look != EOF);
  633. _     }
  634. _ 
  635. _ // eat_return() is intended to save space in Clex::next() -- the
  636. _ //  inline function eat_one() produces quite a lot of code.
  637. _ Clex_sym Clex::
  638. _ eat_return(Clex_sym s)
  639. _     { eat_one(); return s; }
  640. _ 
  641. _ Clex_sym Clex::
  642. _ next()
  643. _     {
  644. _     short val;
  645. _     while (val = look, eat_one(), val != EOF)
  646. _         {
  647. _         char ch = char(val);
  648. _         switch (ch)
  649. _             {
  650. _         case ' ' : continue;
  651. _ 
  652. _         case '_' :
  653. _         case '$' : return ident(ch);
  654. _ 
  655. _         case '0' : case '1' : case '2' : case '3' : case '4' :
  656. _         case '5' : case '6' : case '7' : case '8' : case '9' :
  657. _                    return num(ch);
  658. _ 
  659. _         case ',' : return COMMA_S;
  660. _         case ';' : return SEMI_S;
  661. _         case '[' : if (block_brack) return lbrack(CL_NONE);
  662. _                    else             return LBRACK_S;
  663. _         case ']' : return RBRACK_S;
  664. _         case '{' : return LBRACE_S;
  665. _         case '}' : return RBRACE_S;
  666. _         case '(' : return LPAR_S;
  667. _         case ')' : return RPAR_S;
  668. _         case '~' : return TILDE_S;
  669. _         case '?' : return QUEST_S;
  670. _         case '"' : return quote(ch, QUOTE_S, CL_NONE);
  671. _         case '\'': return quote(ch, APOS_S, CL_NONE);
  672. _ 
  673. _         case '=' :                              // '=', '=='
  674. _             if (look != '=') return AS_S;
  675. _             else  return eat_return(EQ_S);
  676. _ 
  677. _         case ':' :                              // ":", "::"
  678. _             if (look != ':') return COLON_S;
  679. _             else  return eat_return(SCOPE_S);
  680. _ 
  681. _         case '!' :                              // "!", "!="
  682. _             if (look != '=') return BANG_S;
  683. _             else  return eat_return(NE_S);
  684. _ 
  685. _         case '^' :                              // "^", "^="
  686. _             if (look != '=') return CARET_S;
  687. _             else  return eat_return(XORAS_S);
  688. _ 
  689. _         case '*' :                              // '*', '*='
  690. _             if (look != '=') return STAR_S;
  691. _             else  return eat_return(MULAS_S);
  692. _ 
  693. _         case '%' :                              // '%', '%='
  694. _             if (look != '=') return MOD_S;
  695. _             else  return eat_return(MODAS_S);
  696. _ 
  697. _         case '|' :                              //  "|=", "||", "|"
  698. _             if      (look == '|') return eat_return(LOR_S);
  699. _             else if (look == '=') return eat_return(ORAS_S);
  700. _             else                             return VBAR_S;
  701. _ 
  702. _         case '&' :                              // "&", "&=", "&&"
  703. _             if      (look == '&') return eat_return(LAND_S);
  704. _             else if (look == '=') return eat_return(ANDAS_S);
  705. _             else                             return AMPER_S;
  706. _ 
  707. _         case '+' :                              // '+', '++', '+='
  708. _             if      (look == '+') return eat_return(INCRE_S);
  709. _             else if (look == '=') return eat_return(ADDAS_S);
  710. _             else                             return PLUS_S;
  711. _ 
  712. _         case '-' :                              // '--', '-=', '->', '-', 
  713. _             if      (look == '-') return eat_return(DECRE_S);
  714. _             else if (look == '=') return eat_return(SUBAS_S);
  715. _             else if (look == '>') return eat_return(DEREF_S);
  716. _             else                             return MINUS_S;
  717. _ 
  718. _         case '/' :                              // '/*', '//', '/=', '/'
  719. _             if (look == '*')
  720. _                 {
  721. _                 block_comment(CL_NONE);
  722. _                 if (look == EOF)       // almost certainly a mistake:
  723. _                     return ERROR_EOF_S;
  724. _                 else continue;
  725. _                 }
  726. _             else if (look == '/')
  727. _                 { line_comment(); continue; }
  728. _             else if (look == '=') return eat_return(DIVAS_S);
  729. _             else                             return SLASH_S;
  730. _ 
  731. _         case '.' :                              // ".", "..."
  732. _             if (isdigit(look))      return num(ch);
  733. _             else if (look == '.')
  734. _                 {
  735. _                 eat_one();          // check for "..", undefined.
  736. _                 if (look != '.')    return ERROR_UNKN_S;
  737. _                 else    return  eat_return(ELLIP_S);
  738. _                 }
  739. _             else                    return DOT_S;
  740. _ 
  741. _         case '<' :                              // '<=', '<', '<<', '<<='
  742. _             if      (look == '=')   return eat_return(LE_S);
  743. _             else if (look == '<')
  744. _                 {
  745. _                 eat_one();
  746. _                 if  (look != '=')   return SHL_S;
  747. _                 else     return eat_return(SHLAS_S);
  748. _                 }
  749. _             else                    return LT_S;
  750. _ 
  751. _         case '>' :                              // '>=', '>', '>>', '>>='
  752. _             if      (look == '=')   return eat_return(GE_S);
  753. _             else if (look == '>')
  754. _                 {
  755. _                 eat_one();
  756. _                 if  (look != '=')   return SHR_S;
  757. _                 else     return eat_return(SHRAS_S);
  758. _                 }
  759. _             else                    return GT_S;
  760. _ 
  761. _         default:
  762. _             if (isalpha(ch))
  763. _                 return ident(ch);
  764. _             if (ch == '\n')
  765. _                 eoln(CL_NONE);
  766. _             else if (iscntrl(ch))
  767. _                 continue;
  768. _             else
  769. _                 return ERROR_UNKN_S;
  770. _             }
  771. _         }
  772. _ 
  773. _     return EOF_S;
  774. _     }
  775. _ 
  776. _ struct Quickbuf
  777. _     {
  778. _     short len;
  779. _     char line[10240];
  780. _     void put_in(char c) { if (len < sizeof(line)-1) line[len++] = c; }
  781. _     void terminate()    { line[len] = '\0'; }
  782. _     Quickbuf() { len = 0; }
  783. _     };
  784. _ 
  785. _ void Clex::
  786. _ eoln(Clex_mode m)
  787. _     {
  788. _     // assume NL character already eaten.
  789. _     ++line_num;
  790. _     // don't process '#' lines in quotes, comments, or '#' continuations.
  791. _     if (m & (CL_QUOTE|CL_POUND|CL_COMMENT))
  792. _         return;
  793. _ 
  794. _     // eat whitespace
  795. _     while (look != EOF && look != '\n')
  796. _         {
  797. _         if (look == ' ' || iscntrl(char(look))) eat_one();
  798. _         else break;
  799. _         }
  800. _     if (look != '#')
  801. _         return;
  802. _ 
  803. _     // eat the '#' and subsequent whitespace
  804. _     do { eat_one(); if (look == EOF || look == '\n') break; }
  805. _        while (look == ' ' || iscntrl(char(look)));
  806. _ 
  807. _     // collect the '#' line
  808. _     Quickbuf b;
  809. _     do  {   // record line
  810. _         if (look == '\\')       // check for continuation line
  811. _             {
  812. _             eat_one();
  813. _             if (look == '\n') { eat_one(); eoln(m|CL_POUND); }
  814. _             else { b.put_in('\\'); }
  815. _             }
  816. _         else if (look == '/')   // check for comment in '#' line
  817. _             {
  818. _             eat_one();
  819. _             if (look == '*')
  820. _                 {
  821. _                 block_comment(m|CL_POUND);
  822. _                 if (look == EOF) break;
  823. _                 }
  824. _             else if (look == '/') line_comment();
  825. _             else { b.put_in('/'); }
  826. _             }
  827. _         else
  828. _             {
  829. _             if (iscntrl(char(look))) look = ' ';
  830. _             b.put_in(look);
  831. _             eat_one();
  832. _             }
  833. _  
  834. _         } while (look != '\n' && look != EOF);
  835. _     b.terminate();
  836. _ 
  837. _     (void) pound(m, b.line, b.len);     // call virtual handler
  838. _     }
  839. _ 
  840. _ Boolean Clex::
  841. _ pound (Clex_mode m, char* line, short len)
  842. _     {
  843. _     void(m);                // to keep cfront blissful
  844. _     char* cp = line;
  845. _     if (!isdigit(*cp))
  846. _         {
  847. _         if (len < 5) return FALSE;
  848. _         if (strncmp(cp, "line ", 5) != 0)
  849. _             return FALSE;   // don't know what it is
  850. _         cp += 4;
  851. _         while (*cp == ' ') ++cp;
  852. _         if (!isdigit(*cp))
  853. _             return FALSE;
  854. _         }
  855. _ 
  856. _     // # <line> "<filename>"   or    #line <line> "<filename>"
  857. _     line_num = atoi(cp) - 1;    // will be incremented by eoln() later
  858. _     while (isdigit(*cp)) ++cp;
  859. _     while (*cp == ' ')   ++cp;
  860. _     if (*cp == '"')
  861. _         {
  862. _         char* cpq = cp;
  863. _         do { ++cpq; }
  864. _            while (*cpq != '"' && *cpq != '\0');
  865. _         strncpy(filename, cp+1, cpq - cp - 1);
  866. _         filename[cpq - cp - 1] = '\0';
  867. _         }
  868. _ 
  869. _     return TRUE;
  870. _     }
  871. _ 
  872. _ const char* Clex::
  873. _ debug (Clex_sym s)
  874. _     {
  875. _     return (s >= KEYWORD_S) ? keywords[s - KEYWORD_S] : sym_str[s] ;
  876. _     }
  877. %%%EOF%%%
  878. echo "x - kwhash.c"
  879. sed -e 's/^_ //' >kwhash.c <<'%%%EOF%%%'
  880. _ 
  881. _ /* this is a C program */
  882. _ 
  883. _ #include <stdio.h>
  884. _ 
  885. _ static char *keywords[] =
  886. _     {
  887. _     "asm",
  888. _     "auto",
  889. _     "break",
  890. _     "case",
  891. _     "char",
  892. _     "class",
  893. _     "const",
  894. _     "continue",
  895. _     "default",
  896. _     "delete",
  897. _     "do",
  898. _     "double",
  899. _     "else",
  900. _     "enum",
  901. _     "extern",
  902. _     "float",
  903. _     "for",
  904. _     "friend",
  905. _     "goto",
  906. _     "if",
  907. _     "inline",
  908. _     "int",
  909. _     "long",
  910. _     "new",
  911. _     "operator",
  912. _     "overload",
  913. _     "private",
  914. _     "protected",
  915. _     "public",
  916. _     "register",
  917. _     "return",
  918. _     "short",
  919. _     "signed",
  920. _     "sizeof",
  921. _     "static",
  922. _     "struct",
  923. _     "switch",
  924. _     "this",
  925. _     "typedef",
  926. _     "union",
  927. _     "unsigned",
  928. _     "virtual",
  929. _     "volatile",
  930. _     "void",
  931. _     "while"
  932. _     };
  933. _ 
  934. _ #define KW_NUMKEYS (sizeof(keywords)/sizeof(*keywords))
  935. _ 
  936. _ unsigned int hashsize = 137;
  937. _ char** kwhash;
  938. _ typedef unsigned short u_short;
  939. _ 
  940. _ u_short
  941. _ hash(cp, len, a, b, c)
  942. _     unsigned char* cp;
  943. _     u_short len;
  944. _     u_short a, b, c;
  945. _     {
  946. _     return (((u_short)cp[0]         ) ^
  947. _             ((u_short)cp[1]     << a) ^
  948. _             ((u_short)cp[len-1] << b) ^
  949. _              (len               << c)  ) % hashsize;
  950. _     }
  951. _ 
  952. _ int
  953. _ insert(cp, a, b, c)
  954. _     char *cp;
  955. _     u_short a, b, c;
  956. _     {
  957. _     short h;
  958. _ 
  959. _     h = hash(cp, strlen(cp), a, b, c);
  960. _     if (kwhash[h] != NULL)
  961. _         {
  962. _ /*
  963. _         printf("Keyword hash collision: %s, %s\n", kwhash[h], cp);
  964. _ */
  965. _         return 0;
  966. _         }
  967. _     else
  968. _         kwhash[h] = cp;
  969. _     return 1;
  970. _     }
  971. _ 
  972. _ int
  973. _ try(a, b, c)
  974. _     short a, b, c;
  975. _     {
  976. _     short int i;
  977. _     int collisions;
  978. _ 
  979. _     collisions = 0;
  980. _     for (i = 0; i < hashsize; ++i)
  981. _         kwhash[i] = NULL;
  982. _     for (i = 0; i < KW_NUMKEYS; ++i)
  983. _         if (!insert(keywords[i], a, b, c))
  984. _             ++collisions;
  985. _     return collisions;
  986. _     }
  987. _ 
  988. _ main(argc, argv)
  989. _     int argc;
  990. _     char **argv;
  991. _     {
  992. _     int min_collisions;
  993. _     int min_abc = 0;
  994. _     short a, b, c;
  995. _ 
  996. _     if (argc > 1) hashsize = atoi(argv[1]);
  997. _     else
  998. _         {
  999. _         printf("usage: %s <hash_size>\n\t<hash_size> should be prime.\n",
  1000. _                 argv[0]);
  1001. _         exit(-1);
  1002. _         }
  1003. _ 
  1004. _     if (hashsize < KW_NUMKEYS)
  1005. _         {
  1006. _         printf("Hash table is too small.\n");
  1007. _         exit(-1);
  1008. _         }
  1009. _ 
  1010. _     kwhash = (char**) malloc(hashsize * sizeof(char*));
  1011. _     min_collisions = hashsize + 1;
  1012. _     for (a = 0; a <= 10; ++a)
  1013. _         {
  1014. _         for (b = 0; b <= 10; ++b)
  1015. _             {
  1016. _             for (c = 0; c <= 10; ++c)
  1017. _                 {
  1018. _                 int collisions;
  1019. _ 
  1020. _                 collisions = try(a, b, c);
  1021. _                 if (collisions <= min_collisions)
  1022. _                     {
  1023. _                     printf("abc: %03x  Collisions: %2d ",
  1024. _                            ((a<<8)|(b<<4)|c), collisions);
  1025. _                     min_collisions = collisions;
  1026. _                     if (collisions == 0) putchar('*');
  1027. _                     putchar('\n');
  1028. _                     }
  1029. _                 }
  1030. _             }
  1031. _         }
  1032. _     }
  1033. %%%EOF%%%
  1034. echo "x - clex_test.c"
  1035. sed -e 's/^_ //' >clex_test.c <<'%%%EOF%%%'
  1036. _ 
  1037. _ // clex_test -- test code for clex.o
  1038. _ 
  1039. _ #include "clex.h"
  1040. _ 
  1041. _ main()
  1042. _     {
  1043. _     Clex cl = Clex(stdin, TRUE);
  1044. _     Clex_sym s;
  1045. _     do  {
  1046. _         s = cl.next();
  1047. _         printf("%5D ", cl.line_no());
  1048. _         if (s >= KEYWORD_S)
  1049. _             printf(" %s\n", cl.debug(s));
  1050. _         else if (s == IDENT_S ||
  1051. _                  s == NUM_S ||
  1052. _                  s == FLOATNUM_S ||
  1053. _                  s == LBRACK_S ||
  1054. _                  s == APOS_S ||
  1055. _                  s == QUOTE_S )
  1056. _             printf( "      %s \"%s\"\n", cl.debug(s), cl.str());
  1057. _         else
  1058. _             printf( "      %s\n", cl.debug(s));
  1059. _         } while (s > EOF_S);
  1060. _ 
  1061. _     exit(0);
  1062. _     }
  1063. %%%EOF%%%
  1064. echo "Done."
  1065. exit 0
  1066.  
  1067.  
  1068.