home *** CD-ROM | disk | FTP | other *** search
/ OS/2 Shareware BBS: 10 Tools / 10-Tools.zip / lifeos2.zip / LIFE-1.02 / LIB / TOKENIZE.LF < prev    next >
Text File  |  1996-06-04  |  14KB  |  650 lines
  1. %    $Id: tokenizer.lf,v 1.2 1994/12/09 00:26:37 duchier Exp $    
  2. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  3. %
  4. %                          TOKENIZER FOR LIFE
  5. %                             (in wild_life)
  6. %
  7. % This file contains a complete tokenizer for Life programs. The obtained
  8. % tokens are used as inputs by the life parser in life (see parser.lf)
  9. %
  10. % Tokens are of the following types:
  11. %    - variable(X) where X is the name of the variable; 
  12. %    - construct(X) represents a constructor X. 
  13. %      The type of a constructor is a subsort of construct: numb, chaine, or
  14. %      atom. X is the "value" of the atom (string, number, or unevaluated atom)
  15. %    - any syntactic object like "[" or "?", or defined by syntact_object(X)
  16. % 
  17. % The dot may be tokenized in three different ways, depending on the context in
  18. % which it appears:
  19. %   - It is not returned as a token if it occurs inside a floating point
  20. %     number;
  21. %   - It is returned as a syntactic object "." if it is followed by a void
  22. %     character (tab, nl, space, or end_of_file)
  23. %   - it is returned as atom(".") otherwise.
  24. %
  25. % The tokenizer is written as an attribute grammar, using the grammar
  26. % translator. It reads two characters in advance.
  27. %
  28. % Use of this file:
  29. %      tokenize(Filename) ? 
  30. % reads in  the file Filename and writes the obtained tokens in the file
  31. % Filename_toks. 
  32. %    
  33. % All the necessary files are automatically loaded if they are in the same
  34. % directory.  
  35. %
  36. %
  37. % Author: Bruno Dumant
  38. %
  39. % Copyright 1992 Digital Equipment Corporation
  40. % All Rights Reserved
  41. %
  42. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  43.  
  44. module("tokenizer") ?
  45.  
  46. public( atom,construct,numb,chaine,syntact_object,variable,
  47.     tokenize,first_token,rest_token,next_token,
  48.     rest_chars) ?
  49.  
  50.  
  51. %%% load utilities
  52.  
  53. import("accumulators") ?
  54.  
  55. %%% set the right function for handling terminals in the grammar.
  56.  
  57. set_C( token_C) ?
  58.  
  59.  
  60. %%% changed for efficiency
  61.  
  62. acc_info(dcg,Term, Xs, Ys, acc_pred => 'C'(Term,false,Xs,Ys),
  63.      in_name => 0, out_name => rest)?
  64.  
  65.  
  66. token_C([],true,Xs,Ys) -> succeed | Xs = Ys.
  67. token_C([],false,Xs,Ys) -> Xs = Ys.
  68.  
  69. token_C([A],true,Xs,Ys) ->  (`evalin(D) = Ys) | Xs = [A|D].
  70. token_C([A],false,Xs,Ys) -> (  Xs = [A|D], `evalin(D) = Ys ).
  71.  
  72. %%% Types.
  73.  
  74. non_strict(atom) ?
  75. atom <| construct.
  76. numb <| construct.
  77. chaine <| construct.
  78.  
  79. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  80. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  81. %
  82. % Grammar of the tokenizer.
  83. %
  84. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  85. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  86.  
  87. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  88. %
  89. % Main Predicates
  90. %
  91. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  92.  
  93. token(0 => []) :- !, fail.
  94. token(0 => W:[A|B], T, rest => R) :-
  95.     (
  96.         A >= 97 and A =< 122,!,
  97.         non_quoted_atom(SA,0 => W,rest => R),
  98.         cond(  
  99.                 is_syntactic(SA),
  100.             T = SA,
  101.             ( U = str2psi(SA,current_module), T = atom(U))
  102.         )
  103.     ;
  104.         A >= 65 and A=< 90,!,
  105.         variable(V, 0 => W,rest => R),
  106.         T = variable(str2psi(V))
  107.     ;
  108.         cond(   A >= 48 and A =< 57,
  109.                 ( 
  110.             number(N,0 => W,rest => R),
  111.             T = numb(N)
  112.             ),
  113.             str2psi(strcon("tk",int2str(A)))
  114.                    & @(T,0 => W,rest  => R)
  115.         )
  116.     ).
  117.  
  118.  
  119. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  120. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  121. %
  122. % First Character
  123. %
  124. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  125. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  126.  
  127. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  128. %
  129. % variables
  130. %
  131. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  132.  
  133. tk95(T) -->                                      % variables starting with _
  134.     [_],
  135.     var_chars(Y), 
  136.     {   
  137.             (
  138.         Y $== "",!,
  139.         T = atom(@)                     % _ is @
  140.         )                     
  141.     ;
  142.         T = variable(str2psi(strcon("_",Y)))
  143.     } .                  
  144.  
  145. variable(X) -->                                  % variables starting with
  146.     [Y],                                     % a capital letter
  147.     var_chars(Z), 
  148.     { X = strcon(chr(Y),Z) } .
  149.  
  150. var_chars(Z) -->
  151.     simple_atom_chars(Y),
  152.     (
  153.         primes(P),!,                         % variables may end with '
  154.         { Z = strcon(Y,P) }
  155.     ;
  156.         { Z = Y }
  157.     ).
  158.  
  159. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  160. %
  161. % syntactic objects
  162. %
  163. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  164.  
  165.  
  166. tk40( "(") --> [_] .
  167. tk41( ")") --> [_] .
  168. tk63( "?") --> [_] .
  169. tk123( "{") --> [_] .
  170. tk125( "}") --> [_] .
  171.  
  172.  
  173. %%% "." is a syntactic dot, or the special operator ".": the distinction is
  174. %%% done thanks to the following character.
  175.  
  176. tk46( ".",0 => [46])     -->                             %%% end of file
  177.     [_] , ! .
  178. %%%tk46( T,0 => [46,K|L]) -->                  % . is followed by space, tab,
  179. %%%                           % percent or newline.
  180. %%%    { cond(   has_feature(K,void_table),
  181. %%%                  T = ".",
  182. %%%          T = atom(".")
  183. %%%          )
  184. %%%    },
  185. %%%    [_] .
  186.  
  187. tk46( T,0 => [46,K|L]) -->
  188.     (   has_feature(K,void_table),!,
  189.         { T = "." },
  190.         [_]
  191.     ;
  192.         tk46bis(T)
  193.     ).
  194.  
  195. tk46bis(T) -->
  196.     [_],
  197.     op_atom_chars(Z),
  198.     {  
  199.             X = strcon(".",Z),
  200.         ( is_syntactic(X),!,
  201.           T = X
  202.         ;
  203.           U = str2psi(X,current_module),
  204.           T = atom(U)
  205.         )
  206.     } .
  207.  
  208. tk91(T) --> 
  209.     [_],
  210.     ( 
  211.         [124],!,
  212.         (
  213.         [93],!, { T = "[|]"}
  214.         ;
  215.             {T = "[|"}
  216.         )
  217.     ;
  218.         {T = "["}
  219.     ) .
  220.     
  221. tk93( "]") --> [_] .
  222.  
  223. %% special case: "|]"
  224.  
  225. tk124(T) -->
  226.     [_],
  227.     (
  228.         [93],!, { T = "|]"}
  229.     ;
  230.         op_atom_chars(Z),
  231.         {  
  232.             X = strcon("|",Z),
  233.         cond(   is_syntactic(X),
  234.                 T = X,
  235.             ( U = str2psi(X,current_module), T = atom(U))
  236.             )
  237.         } 
  238.     ) . 
  239.         
  240. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  241. %
  242. % constructors
  243. %
  244. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  245.  
  246.  
  247. %%% simple atoms
  248.        
  249. non_quoted_atom(X) --> 
  250.     [Y], 
  251.     simple_atom_chars(Z), 
  252.     { X = strcon(chr(Y),Z) } .
  253.  
  254. %%% quoted atoms
  255.  
  256. tk39(T) --> 
  257.     [_],
  258.         quoted_atom_end(X),
  259.     {  cond(   is_syntactic(X),
  260.                T = X,
  261.            ( U = str2psi(X,current_module), T = atom(U) )
  262.            )
  263.     } .
  264.  
  265.  
  266. %%% Numbers: it is necessary to read two characters in advance.
  267.  
  268. number(X, 0 => W, rest => R) :-
  269.     digits( V1, 0 => W, rest => R1),
  270.     (
  271.         R1 = [46,D|R2], 
  272.         digits(0 => [D|R2],V2,length => L2,rest => R3), 
  273.         R4 = evalin(R3),
  274.         Vint = V1 + V2 * 10^(-L2),
  275.         !
  276.     ;
  277.         Vint = V1,
  278.         R4 = R1
  279.  
  280.     ),
  281.     (
  282.         R4 = [{101;69}|R5],!,    %% e or E
  283.         exponent(E,0 => evalin(R5),rest=> R),
  284.         X = Vint * 10^(E)
  285.     ;
  286.         X = Vint, R4 = R
  287.     ).
  288.  
  289.  
  290. %%% Strings
  291.  
  292. tk34(chaine(X)) --> 
  293.     [_],
  294.         char_chaine_end(X) .
  295.  
  296. %%% op_atoms
  297.  
  298. gen_op_char_ass_pred_def(Char) :-
  299.     S = chr(Char),
  300.     str2psi(strcon("tk",int2str(Char))) = PredName,
  301.     Head = PredName & @(T),
  302.     (
  303.         Head --> 
  304.              [_],
  305.          op_atom_chars(Z),
  306.          {  
  307.              X = strcon(S,Z),
  308.              cond(   is_syntactic(X),
  309.                      T = X,
  310.                  ( U = str2psi(X,current_module), T = atom(U))
  311.              )
  312.          } 
  313.     ).
  314.     
  315. maprel(gen_op_char_ass_pred_def,
  316.        [33,35,36,37,38,42,43,45,47,58,60,61,62,92,94,126]) ?
  317.  
  318.  
  319. %%% special cases: @ , ; `
  320.  
  321. tk64( atom(@)) --> [_] .
  322. tk96( atom(`)) --> [_] .
  323. tk44( atom(,)) --> [_] .    
  324. tk59( atom(;)) --> [_] .
  325.  
  326. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  327. %
  328. % Other Characters
  329. %
  330. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  331.  
  332.  
  333. %%% void chars
  334.  
  335. void_chars -->                                  % space, tab, new line
  336.     [{9;10;32}],!,
  337.     void_chars .   
  338. void_chars -->                                  % comments
  339.     [37],!, 
  340.     comment_chars, 
  341.     void_chars .
  342. void_chars -->                                  % nested comments
  343.     nested_comments,
  344.     void_chars .
  345. void_chars --> [] .
  346.  
  347. comment_chars --> [10], ! .                     % a comment ends with a nl.
  348. comment_chars --> [X],!, comment_chars .
  349. comment_chars --> [] .                          % nothing left to read
  350.  
  351.  
  352. %%% nested_comments
  353.  
  354. nested_comments( 0 => [47,42|_]) -->
  355.     [_],[_],
  356.     end_nested_comments .
  357.  
  358. end_nested_comments( 0 => [42,47|_]) -->
  359.         [_],[_],! .
  360.  
  361. end_nested_comments -->
  362.     (
  363.         nested_comments
  364.     ;
  365.         [_]
  366.     ),
  367.     end_nested_comments .
  368.  
  369. %%% simple atom characters
  370.  
  371. simple_atom_chars(Z) --> 
  372.     simple_atom_char(X), !,
  373.     simple_atom_chars(Y), 
  374.     { Z = strcon(X,Y)} .
  375. simple_atom_chars("") --> [] .
  376.  
  377.  
  378. simple_atom_char(X, 0 => [Y|R1], rest => R2) :- 
  379. %%    Y >= 48 and Y =< 57        % chiffre
  380. %%     or Y >= 65 and Y =< 90        % majuscule
  381. %%     or Y =:= 95                   % underscore
  382. %%     or Y >= 97 and Y =< 122,      % minuscule 
  383. %%     R2 = evalin(R1),
  384. %%     X = chr(Y).
  385.         has_feature(Y,simple_atom_table,X),
  386.     R2 = evalin(R1).
  387.  
  388. at_least_1_simple_atom_char(Z) -->
  389.     simple_atom_char(X), !,
  390.     simple_atom_chars(Y), 
  391.     { Z = strcon(X,Y) } .
  392.  
  393. %%% primes
  394.  
  395. primes(P) -->
  396.     [39],
  397.     (
  398.         primes(Q), {P = strcon("'",Q), !}
  399.     ;
  400.         {P = "'"}
  401.     ). 
  402.  
  403. %%% quoted atoms
  404.  
  405. quoted_atom_end(X) --> 
  406.     [39], !,
  407.     ( 
  408.         [39], !,quoted_atom_end(Y), X = strcon("'",Y) 
  409.     ; 
  410.         { X = "" }
  411.     ) .
  412.  
  413. quoted_atom_end(X) --> 
  414.     any_char(Y),
  415.     quoted_atom_end(Z),
  416.     {  X = strcon(Y,Z)} .
  417.  
  418.  
  419. %%% numbers
  420.  
  421. digits(V, length=>L) --> 
  422.     digit( V1), 
  423.     ( digits(V2,length => L2),!,
  424.       { L = L2+1, V = V1*10^L2 + V2} ;
  425.           { V = V1, L = 1}) .
  426.  
  427. digit(0 => [48+N|R], N, rest => Rest)  :-
  428.     N =< 9 and N >= 0,
  429.     Rest = evalin(R).
  430.  
  431. exponent(V) --> sign(S), digits(V1), {!, V = S*V1} .
  432.  
  433. sign(1) --> [] .
  434. sign(-1) --> [45], ! .
  435. sign(1) --> [43] .
  436.  
  437.  
  438. %%% strings 
  439.  
  440. char_chaine_end(X) --> 
  441.     [34], !,
  442.     ( [34], !,char_chaine_end(Y), {X = strcon("""",Y)} ; 
  443.           { X = "" }) .
  444. char_chaine_end(X) --> 
  445.     any_char(Y),
  446.     char_chaine_end(Z),
  447.     { X = strcon(Y,Z)} .
  448.  
  449.  
  450. %%% characters for "operators"
  451.  
  452. op_atom_char(X, 0=> [Y|R1], rest => R2) :- 
  453.     has_feature(Y,op_chars_table,X),
  454.     R2 = evalin(R1).
  455.  
  456. op_atom_chars(X) --> 
  457.     op_atom_char(Y),!, 
  458.     op_atom_chars(Z),
  459.     { X = strcon(Y,Z)} .
  460. op_atom_chars("") -->
  461.     [] .
  462.  
  463.  
  464. %%% any character
  465.  
  466. any_char(Y) -->
  467.     [X], { Y = chr(X) } .
  468.  
  469. %
  470. % reset C function
  471. %
  472.  
  473. reset_C ?
  474.  
  475. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  476. %
  477. % characters tables
  478. %
  479. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  480. gen_char_table(Table,[A|B]) :-
  481.     !,
  482.     Table.A <<- chr(A),
  483.     gen_char_table(Table,B).
  484. gen_char_table.
  485.  
  486. persistent(void_table) ?
  487. gen_char_table(void_table,[9,10,32,37]) ?
  488.  
  489. persistent(simple_atom_table) ?
  490. gen_char_table(simple_atom_table,
  491.           [48,49,50,51,52,53,54,55,56,57,65,66,67,68,69,70,
  492.        71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,
  493.        89,90,95,97,98,99,100,101,102,103,104,105,106,107,
  494.        108,109,110,111,112,113,114,115,116,117,118,119,120,121,122]) ?
  495.  
  496. persistent(op_chars_table) ?
  497. gen_char_table(op_chars_table,[33,35,36,37,38,42,43,45,46,47,
  498.                            58,60,61,62,92,94,124,126]) ?
  499.  
  500.  
  501. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  502. %
  503. %  definition of new syntactic objects
  504. %
  505. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  506.  
  507. is_syntactic(X) -> has_feature(X,syntact_objects_table).
  508.  
  509. persistent(syntact_objects_table) ?
  510.  
  511. syntact_object(X) :-
  512.     is_value(X),!,
  513.     nl_err,
  514.     write_err(
  515.         "*** Error: numbers or strings cannot be syntactic objects."),
  516.     nl_err
  517.     ;
  518.     syntact_objects_table.X = true.
  519.  
  520. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  521. %
  522. % char handler
  523. %
  524. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  525.  
  526. persistent(read_tok) ?
  527. read_tok <<- false ?
  528. persistent(rest_chars) ?
  529. rest_chars <<- [] ?
  530. persistent(rest_token) ?
  531. rest_token <<- none ?
  532.  
  533. next_char -> 
  534.     L 
  535.     |   
  536.         get(X),
  537.     cond( X :=< end_of_file , 
  538.           L=[] ,
  539.               L = [X|T] ),
  540.         T = `next_char.
  541.  
  542. next_token -> L |
  543.          (
  544.           cond( R:copy_term(rest_chars) = [A,B],
  545.             Chars = [A,B|`next_char],
  546.            Chars = R ),
  547.              call_once(read_new_token(Tok, Chars)) = TT,
  548.          (
  549.          TT :== false, !, fail 
  550.          ; 
  551.          Tok :== none, !,
  552.          L = []
  553.          ;
  554.              rest_token <<- `Tok,
  555.          fail 
  556.          )
  557.      ;
  558.          L = [copy_term(rest_token)|`next_token]
  559.      ).
  560.  
  561.  
  562.  
  563. first_token ->
  564.     L
  565.     |   
  566.         FC = next_char,
  567.         NC = evalin(FC),                   %% 2 characters must be read
  568.         read_new_token(Tok,NC),
  569.     cond( Tok :== none,
  570.           L = [],
  571.           L = [Tok|T]),
  572.     T = `next_token.
  573.  
  574.  
  575. read_new_token( Tok, X) :- 
  576.     void_chars(0 => X, rest => R1),!,
  577.     (
  578.         R1 = [], !, Tok = none
  579.     ;
  580.         token( 0 => R1, Tok, rest => R2),
  581.         ( 
  582.         R2 = [A,B|R],!,
  583.         rest_chars <<- [A,B]
  584.         ;
  585.         rest_chars <<- R2
  586.         )
  587.     ).
  588.  
  589. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  590. %
  591. % Interface Predicates
  592. %
  593. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  594.  
  595. tokens(true,X,S1) :-
  596.     read_tok <<- false,
  597.     void_chars(0 => X, rest => R1),!,
  598.     (
  599.         R1 = [], !,
  600.         open_out("stdout",S),
  601.         nl,nl,
  602.         write("*** File '",S1.input_file_name,"' tokenized"),
  603.         nl
  604.     ;
  605.         (  
  606.         token( 0 => R1, T, rest => R2),
  607.         nl,writeq(T),
  608.         read_tok <<- true,
  609.         cond( R2 = [A,B|_],
  610.               rest_chars <<- [A,B],
  611.               rest_chars <<- R2),
  612.         fail
  613.         ;
  614.             cond( R:copy_term(rest_chars) = [A,B],
  615.                Chars = [A,B|`next_char],
  616.               Chars = R ),
  617.         tokens(read_tok,Chars,S1)
  618.         )
  619.     ).
  620. tokens(false,_,S1) :- 
  621.     open_out("stdout",S),
  622.     nl_err,nl_err,
  623.     write_err("*** Token error near line ",S1.line_count,
  624.               " in file '",S1.input_file_name,"'"),
  625.     nl_err.
  626.  
  627. tokenize(File:string) :- 
  628.     open_in(File,S1),
  629.     open_out(strcon(File,"_toks"),S2),
  630.     FC = next_char,
  631.     NC = evalin(FC),
  632.     tokens(true,NC,S1),
  633.     close(S1),
  634.     close(S2).
  635.  
  636. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  637. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  638.  
  639.  
  640.  
  641.  
  642.  
  643.  
  644.  
  645.  
  646.  
  647.  
  648.  
  649.  
  650.