home *** CD-ROM | disk | FTP | other *** search
/ Atari FTP / ATARI_FTP_0693.zip / ATARI_FTP_0693 / Tex / Tex29 / StTeXsrc.zoo / src / token.c < prev    next >
C/C++ Source or Header  |  1988-03-13  |  15KB  |  522 lines
  1.  
  2. /*
  3.  * @(#)token.c 2.6 EPA
  4.  *
  5.  * Copyright 1987,1988 Pat J Monardo
  6.  *
  7.  * Redistribution of this file is permitted through
  8.  * the specifications in the file COPYING.
  9.  *
  10.  * 
  11.  */
  12.  
  13. #include "tex.h"
  14. #include "scan.h"
  15. #include "box.h"
  16. #include "cond.h"
  17. #include "expand.h"
  18. #include "align.h"
  19. #include "tokenstack.h"
  20. #include "token.h"
  21.  
  22. tok     cur_tok;
  23. hword   cur_cmd;
  24. hword   cur_chr;
  25. ptr     cur_cs;
  26.  
  27. /*
  28.  *  This is the TeX segment
  29.  */
  30.  
  31. tok     tok_mem[TOK_MAX-TOK_MIN+1];
  32. ptr     tok_link[TOK_MAX-TOK_MIN+1];
  33. ptr     tok_head;
  34. ptr     tok_low;
  35. ptr     tok_end;
  36. int     tok_used;
  37.  
  38. ptr     par_loc;
  39. tok     par_token;
  40.  
  41. bool    force_eof;
  42.  
  43. get_token ()
  44. {
  45.     no_new_control_sequence = FALSE;
  46.     get_next();
  47.     no_new_control_sequence = TRUE;
  48.     if (cur_cs == 0)
  49.         cur_tok = cur_cmd * 0400 + cur_chr;
  50.     else cur_tok = CS_TOKEN_FLAG + cur_cs;
  51. }
  52.  
  53. #define reduce_expanded_cc() \
  54.     {if (buffer[k] == cur_chr && \
  55.         cat == SUP_MARK && k < limit) { \
  56.         cur_chr = buffer[k + 1]; \
  57.         if (cur_chr < 0100) \
  58.             buffer[k - 1] = cur_chr + 0100; \
  59.         else buffer[k - 1] = cur_chr - 0100; \
  60.         limit -= 2; \
  61.         first -= 2; \
  62.         while (k <= limit) { \
  63.             buffer[k] = buffer[k + 2]; \
  64.             incr(k);} \
  65.         goto start_cs;}}
  66.         
  67. #define any_state(CAT) \
  68.     case MID_LINE + CAT: \
  69.     case SKIP_BLANKS + CAT: \
  70.     case NEW_LINE + CAT
  71.             
  72. #define delims(CAT) \
  73.     case MATH_SHIFT + CAT: \
  74.     case TAB_MARK + CAT: \
  75.     case MAC_PARAM + CAT: \
  76.     case SUB_MARK + CAT: \
  77.     case LETTER + CAT: \
  78.     case OTHER_CHAR + CAT
  79.  
  80. #define mid_line(CAT) \
  81.     case MID_LINE + CAT
  82.  
  83. #define new_line(CAT) \
  84.     case NEW_LINE + CAT
  85.  
  86. #define skip_blanks(CAT) \
  87.     case SKIP_BLANKS + CAT
  88.  
  89. get_next ()
  90. {
  91.     int     k;
  92.     tok     t;
  93.     int     cat;
  94.  
  95. restart:
  96.     cur_cs = 0;
  97.     if (state != TOKEN_LIST) {
  98. reread:
  99.         if (loc <= limit) {
  100.             cur_chr = buffer[loc];
  101.             incr(loc);
  102. reswitch:
  103.             cur_cmd = cat_code(cur_chr);
  104.             switch (state + cur_cmd)
  105.             {
  106.             any_state(IGNORE):
  107.             skip_blanks(SPACER):
  108.             new_line(SPACER):
  109.                 goto reread;
  110.  
  111.             any_state(ESCAPE):
  112.                 if (loc > limit) {
  113.                     cur_cs = NULL_CS;
  114.                 } else {
  115.                 start_cs: 
  116.                     k = loc;
  117.                     cur_chr = buffer[k];
  118.                     cat = cat_code(cur_chr);
  119.                     incr(k);
  120.                     if (cat == LETTER)
  121.                         state = SKIP_BLANKS;
  122.                     else if (cat == SPACER)
  123.                         state = SKIP_BLANKS;
  124.                     else state = MID_LINE;
  125.                     if (cat == LETTER && k <= limit) {
  126.                         do {
  127.                             cur_chr = buffer[k];
  128.                             incr(k);
  129.                             cat = cat_code(cur_chr);
  130.                         } while (cat == LETTER && k <= limit);
  131.                         reduce_expanded_cc();
  132.                         if (cat != LETTER)
  133.                             decr(k);
  134.                         if (k > loc + 1) {
  135.                             cur_cs = id_lookup(loc, k - loc);
  136.                             loc = k;
  137.                             goto found;
  138.                         }
  139.                     } else {
  140.                         reduce_expanded_cc();
  141.                     }
  142.                     cur_cs = SINGLE_BASE + buffer[loc];
  143.                     incr(loc);
  144.                 }
  145.                 found:
  146.                 cur_cmd = eq_type(cur_cs);
  147.                 cur_chr = equiv(cur_cs);
  148.                 if (cur_cmd >= OUTER_CALL)
  149.                     check_outer_validity();
  150.                 break;
  151.                         
  152.             any_state(ACTIVE_CHAR):
  153.                 cur_cs = cur_chr + ACTIVE_BASE;
  154.                 cur_cmd = eq_type(cur_cs);
  155.                 cur_chr = equiv(cur_cs);
  156.                 state = MID_LINE;
  157.                 if (cur_cmd >= OUTER_CALL)
  158.                     check_outer_validity();
  159.                 break;
  160.             
  161.             any_state(SUP_MARK):
  162.                 if (cur_chr == buffer[loc] && loc < limit) {
  163.                     if (buffer[loc + 1] < 0100)
  164.                         cur_chr = buffer[loc + 1] + 0100;
  165.                     else cur_chr = buffer[loc + 1] - 0100;
  166.                     loc += 2;
  167.                     goto reswitch;
  168.                 }
  169.                 state = MID_LINE;
  170.                 break;
  171.  
  172.             any_state(INVALID_CHAR):
  173.                 print_err("Text line contains an invalid character");
  174.                 help_funny();
  175.                 deletions_allowed = FALSE;
  176.                 error();
  177.                 deletions_allowed = TRUE;
  178.                 goto restart;
  179.             
  180.             mid_line(SPACER):
  181.                 state = SKIP_BLANKS;
  182.                 cur_chr = ' ';
  183.                 break;
  184.             
  185.             mid_line(CAR_RET):
  186.                 loc = limit + 1;
  187.                 cur_cmd = SPACER;
  188.                 cur_chr = ' ';
  189.                 break;
  190.             
  191.             skip_blanks(CAR_RET):
  192.             any_state(COMMENT):
  193.                 loc = limit + 1;
  194.                 goto reread;
  195.  
  196.             new_line(CAR_RET):
  197.                 loc = limit + 1;
  198.                 cur_cs = par_loc;
  199.                 cur_cmd = eq_type(cur_cs);
  200.                 cur_chr = equiv(cur_cs);
  201.                 if (cur_cmd >= OUTER_CALL)
  202.                     check_outer_validity();
  203.                 break;
  204.             
  205.             mid_line(LEFT_BRACE):
  206.                 incr(align_state);
  207.                 break;
  208.  
  209.             skip_blanks(LEFT_BRACE):
  210.             new_line(LEFT_BRACE):
  211.                 state = MID_LINE;
  212.                 incr(align_state);
  213.                 break;
  214.             
  215.             mid_line(RIGHT_BRACE):
  216.                 decr(align_state);
  217.                 break;
  218.             
  219.             skip_blanks(RIGHT_BRACE):
  220.             new_line(RIGHT_BRACE):
  221.                 state = MID_LINE;
  222.                 decr(align_state);
  223.                 break;
  224.             
  225.             delims(SKIP_BLANKS):
  226.             delims(NEW_LINE):
  227.                 state = MID_LINE;
  228.                 break;
  229.  
  230.             default:
  231.                 break;
  232.             }
  233.         } else {
  234.             state = NEW_LINE; 
  235.             if (name > 17) {
  236.                 incr(line);
  237.                 first = start;
  238.                 if (!force_eof) {
  239.                     if (input_ln(cur_file, TRUE))
  240.                         firm_up_the_line();
  241.                     else force_eof = TRUE;
  242.                 }
  243.                 if (force_eof) {
  244.                     print_char(')');
  245.                     force_eof = FALSE;
  246.                     update_terminal();
  247.                     end_file_reading();
  248.                     check_outer_validity();
  249.                     goto restart;
  250.                 }
  251.                 if (end_line_char < 0 || end_line_char > 127)
  252.                     decr(limit);
  253.                 else buffer[limit] = end_line_char;
  254.                 first = limit + 1;
  255.                 loc = start;
  256.             } else {
  257.                 if (!terminal_input) {
  258.                     cur_cmd = 0;
  259.                     cur_chr = 0;
  260.                     return;
  261.                 }
  262.                 if (input_ptr > 0) {
  263.                     end_file_reading();
  264.                     goto restart;
  265.                 }
  266.                 if (selector < LOG_ONLY)
  267.                     open_log_file();
  268.                 if (interaction > NONSTOP_MODE) {
  269.                     if (end_line_char < 0 || end_line_char > 255)
  270.                         incr(limit);
  271.                     if (limit == start)
  272.                         print_nl("(Please type a command or say `\\end')");
  273.                     print_ln();
  274.                     first = start;
  275.                     prompt_input("*");
  276.                     limit = last;
  277.                     if (end_line_char < 0 || end_line_char > 127)
  278.                         decr(limit);
  279.                     else buffer[limit] = end_line_char;
  280.                     first = limit + 1;
  281.                     loc = start;
  282.                 } else
  283.                     fatal_error(
  284.                         "*** (job aborted, no legal \\end found)");
  285.             }
  286.             check_interrupt();
  287.             goto reread;
  288.         }
  289.     } else {
  290.         if (loc != NULL) {
  291.             t = token(loc);
  292.             loc = token_link(loc);
  293.             if (t >= CS_TOKEN_FLAG) {
  294.                 cur_cs = t - CS_TOKEN_FLAG;
  295.                 cur_cmd = eq_type(cur_cs);
  296.                 cur_chr = equiv(cur_cs);
  297.                 if (cur_cmd >= OUTER_CALL) {
  298.                     if (cur_cmd == DONT_EXPAND) {
  299.                         cur_cs = token(loc) - CS_TOKEN_FLAG;
  300.                         loc = NULL;
  301.                         cur_cmd = eq_type(cur_cs);
  302.                         cur_chr = equiv(cur_cs);
  303.                         if (cur_cmd > MAX_COMMAND) {
  304.                             cur_cmd = RELAX;
  305.                             cur_chr = NO_EXPAND_FLAG;
  306.                         }
  307.                     } else {
  308.                         check_outer_validity();
  309.                     }
  310.                 }
  311.             } else {
  312.                 cur_cmd = t / 0400;
  313.                 cur_chr = t % 0400;
  314.                 switch (cur_cmd)
  315.                 {
  316.                 case LEFT_BRACE:
  317.                     incr(align_state);
  318.                     break;
  319.  
  320.                 case RIGHT_BRACE:
  321.                     decr(align_state);
  322.                     break;
  323.  
  324.                 case OUT_PARAM:
  325.                     begin_token_list(
  326.                         param_stack[param_start + cur_chr - 1],
  327.                         PARAMETER
  328.                     );
  329.                     goto restart;
  330.  
  331.                 default:
  332.                     break;
  333.                 }
  334.             }
  335.         } else {
  336.             end_token_list();
  337.             goto restart;
  338.         }
  339.     }
  340.     if (cur_cmd <= CAR_RET &&
  341.         cur_cmd >= TAB_MARK &&
  342.         align_state == 0) {
  343.         if (scanner_status == ALIGNING)
  344.             fatal_error("(interwoven alignment preambles are not allowed)");
  345.         cur_cmd = extra_info(cur_align);
  346.         extra_info(cur_align) = cur_chr;
  347.         if (cur_cmd == OMIT)
  348.             begin_token_list(omit_template, (qword) V_TEMPLATE);
  349.         else begin_token_list((ptr) v_part(cur_align), (qword) V_TEMPLATE);
  350.         align_state = 1000000;
  351.         goto restart;
  352.     }
  353. }
  354.  
  355. check_outer_validity ()
  356. {
  357.     ptr     p;
  358.     ptr     q;
  359.  
  360.     if (scanner_status != NORMAL) {
  361.         deletions_allowed = FALSE;
  362.         if (cur_cs != 0) {
  363.             if (state == TOKEN_LIST || name < 1 || name > 17) {
  364.                 p = new_token();
  365.                 token(p) = CS_TOKEN_FLAG + cur_cs;
  366.                 back_list(p);
  367.             }
  368.             cur_cmd = SPACER;
  369.             cur_chr = ' ';
  370.         }
  371.         if (scanner_status > SKIPPING) {
  372.             runaway();
  373.             if (cur_cs == 0) {
  374.                 print_err("File ended");
  375.             } else {
  376.                 cur_cs = 0;
  377.                 print_err("Forbidden control sequence found");
  378.             }
  379.             print(" while scanning ");
  380.             p = new_token();
  381.             switch (scanner_status)
  382.             {
  383.             case DEFINING:
  384.                 print("definition");
  385.                 token(p) = RIGHT_BRACE_TOKEN + '}';
  386.                 break;
  387.  
  388.             case MATCHING:
  389.                 print("use");
  390.                 token(p) = par_token;
  391.                 long_state = OUTER_CALL;
  392.                 break;
  393.  
  394.             case ALIGNING:
  395.                 print("preamble");
  396.                 token(p) = RIGHT_BRACE_TOKEN + '}';
  397.                 q = p;
  398.                 p = new_token();
  399.                 token_link(p) = q;
  400.                 token(p) = CS_TOKEN_FLAG + FROZEN_CR;
  401.                 align_state = -1000000;
  402.                 break;
  403.  
  404.             case ABSORBING:
  405.                 print("text"); 
  406.                 token(p) = RIGHT_BRACE_TOKEN + '}';
  407.                 break;
  408.             }
  409.             ins_list(p);
  410.             print(" of ");
  411.             sprint_cs(warning_index);
  412.             help_scanner();
  413.             error();
  414.         } else {
  415.             print_err("Incomplete ");
  416.             print_cmd_chr(IF_TEST, cur_if);
  417.             print("; all text was ignored after line ");
  418.             print_val(skip_line);
  419.             help_skif();
  420.             if (cur_cs != 0)
  421.                 cur_cs = 0;
  422.             else
  423.                 help_line[0] = 
  424.                     "The file ended while I was skipping conditional text.";
  425.             cur_tok = CS_TOKEN_FLAG + FROZEN_FI;
  426.             ins_error();
  427.         }
  428.         deletions_allowed = TRUE;
  429.     }
  430. }
  431.  
  432. firm_up_the_line ()
  433. {
  434.     int     k;
  435.  
  436.     limit = last;
  437.     if (pausing > 0 && interaction > NONSTOP_MODE) {
  438.         wake_up_terminal();
  439.         print_ln();
  440.         if (start < limit) {
  441.             for (k = start; k < limit; incr(k))
  442.                 print_char(buffer[k]);
  443.         }
  444.         first = limit;
  445.         prompt_input("=>");
  446.         if (last > first) {
  447.             for (k = first; k < last; incr(k))
  448.                 buffer[k + start - first] = buffer[k];
  449.             limit = start + last - first;
  450.         }
  451.     }
  452. }
  453.  
  454. ptr
  455. new_token ()
  456. {
  457.     ptr     p;
  458.  
  459.     p = tok_head;
  460.     if (p != NULL) {
  461.         tok_head = token_link(tok_head);
  462.     } else if (tok_end < TOK_MAX) {
  463.         incr(tok_end);
  464.         p = tok_end;
  465.     } else {
  466.         decr(tok_low);
  467.         p = tok_low;
  468.         if (tok_low <= TOK_MIN) {
  469.             runaway();
  470.             overflow("token memory size", TOK_MAX - TOK_MIN + 1);
  471.         }
  472.     }
  473.     token_link(p) = NULL;
  474. #ifdef STAT
  475.     incr(tok_used);
  476. #endif
  477.     return p;
  478. }
  479.  
  480. init_tok_mem()
  481. {
  482.     int     i;
  483.  
  484. #ifdef INIT
  485.     tok_head = NULL;
  486.     tok_end = TOK_TOP;
  487.     tok_low = tok_high;
  488.     for (i = TOK_TOP; i > tok_low; decr(i)) {
  489.         token(i) = 0;
  490.         token_link(i) = NULL;
  491.     }
  492.     token(omit_template) = END_TEMPLATE_TOKEN;
  493.     tok_used = tok_usage;
  494. #endif
  495.  
  496. }
  497.  
  498. /*
  499.  *  Help text
  500.  */
  501.  
  502. help_scanner ()
  503. {
  504.     help4("I suspect you have forgotten a `}', causing me",
  505.     "to read past where you wanted me to stop.",
  506.     "I'll try to recover; but if the error is serious,",
  507.     "you'd better type `E' or `X' now and fix your file.");
  508. }
  509.  
  510. help_funny ()
  511. {
  512.     help2("A funny symbol that I can't read has just been input.",
  513.     "Continue, and I'll forget that it ever happened.");
  514. }
  515.  
  516. help_skif ()
  517. {
  518.     help3("A forbidden control sequence occurred in skipped text.",
  519.     "This kind of error happens when you say `\\if...' and forget",
  520.     "the matching `\\fi'. I've inserted a `\\fi'; this might work.");
  521. }
  522.