home *** CD-ROM | disk | FTP | other *** search
/ Programmer 7500 / MAX_PROGRAMMERS.iso / CLIPPER / MISC / AIE8911.ZIP / NEXT_TOK.ARI < prev    next >
Encoding:
Text File  |  1989-05-31  |  6.1 KB  |  249 lines
  1.  
  2.  
  3.  
  4. %%%%%%%%%% end prepcomp generated declarations %%%%%%%%%%%%%%%%%%%%
  5.  
  6.  
  7. % :- module next_tok.
  8.  
  9. :- extrn next_tok_trace / 0 : interp.
  10.  
  11.  
  12.  
  13. next_tok_trace( X) :-
  14.      call( next_tok_trace),
  15.      !,
  16.      trace_message(X).
  17.  
  18. next_tok_trace( _).
  19.  
  20.  
  21. %%%%%%%%%%%%%%%%%%%%% tokens_between_sep %%%%%%%%%%%%%%%%%%%%%%%%%%%
  22.  
  23. tokens_between_sep( _   , In, _   ) :-
  24.      next_tok_trace([$e tokens_between_sep , In = $, In]),
  25.      fail.
  26.  
  27. tokens_between_sep( []  , [], []  ) :-  !.
  28.  
  29. tokens_between_sep(Tokens  , In, Out ) :-
  30.            ( string( In) ; atom(In)),
  31.            !,
  32.            list_text( List, In),
  33.            tokens_between_sep( Tokens  , List, Out ) .
  34.  
  35. tokens_between_sep( [H|T] ) -->
  36.          next_token_between_sep( H ),
  37.          !,
  38.          tokens_between_sep(T ).
  39. tokens_between_sep( [], _, _) :-!.
  40.  
  41.  
  42. %%%%%%%%%%%%%%%% next_token_between_sep %%%%%%%%%%%%%%%%%%%%%%%%%%%
  43. /* This returns the next token, where a token is whatever is between
  44.    separators or commas.
  45.  
  46. CALL: next_token_between_sep(Token,
  47.                                Input_list_of_chars,
  48.                                Left_over_tail_of_input)
  49.  
  50.  
  51.  
  52. */
  53. :- mode next_token_between_sep( ? , +, ?).
  54.  
  55. next_token_between_sep( _   , In, _   ) :-
  56.      next_tok_trace([$e next_token_between_sep , In = $, In]),
  57.      fail.
  58.  
  59. next_token_between_sep(   _   , [], _   ) :-
  60.            !,
  61.            fail.
  62.  
  63. next_token_between_sep(Token  , In, Out ) :-
  64.            ( string( In) ; atom(In)),
  65.            !,
  66.            list_text( List, In),
  67.            next_token_between_sep( Token  , List, Out ) .
  68.  
  69. next_token_between_sep(Token) -->
  70.            separator_or_comma(_),
  71.            !,
  72.            next_token_between_sep(Token ).
  73.  
  74. next_token_between_sep($($) -->
  75.           [`(],
  76.           !.
  77.  
  78. next_token_between_sep($)$) -->
  79.           [`)],
  80.           !.
  81.  
  82. next_token_between_sep(Token) -->
  83.            [X],
  84.            !,
  85.            between_sep_tail(Y),
  86.            {list_text( [X|Y], Token)}.
  87.  
  88. between_sep_tail( _   , In, _   ) :-
  89.      next_tok_trace([$e between_sep_tail , In = $, In]),
  90.      fail.
  91.  
  92. between_sep_tail( []   ) -->
  93.            separator_or_comma(_),
  94.            !.
  95.  
  96. between_sep_tail( [], Rest, Rest   ) :-
  97.            Rest = [H|_],
  98.            ( H == `(; H==`) ),
  99.            !.
  100.  
  101. between_sep_tail( [H | T ]   ) -->
  102.            [ H ],
  103.            !,
  104.            between_sep_tail(  T ).
  105.  
  106.  
  107. between_sep_tail( [  ]   ) --> !.
  108.  
  109. separator_or_comma(X) -->
  110.      separator(X), !.
  111. separator_or_comma(X) -->
  112.       comma(X), !.
  113.  
  114. comma(X) --> [X], {X = `,}.
  115.  
  116. %%%%%%%%%%%%%%%%%%%% next_token %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  117. /*
  118. next_token(Token,
  119.            Input_list_of_chars,
  120.            Left_over_tail_of_input)
  121.  
  122.    returns the next Token in Input_list_of_chars.
  123.  
  124.    In more detail,
  125.  
  126.    next_token(Token) --> returns a single typed token, which is either
  127.           digits(D) where D is a string of digits
  128.           alphanum(A) where A is an alphanum string
  129.           eof($EOF$) when an end of file char is encountered
  130.                      currently ASCII 26
  131.           char(C) where C is a 1-char string containing a
  132.                   non-separator non-alphanum char
  133.  
  134. The  4-argument versions are called like this:
  135.  
  136. next_token(Token,
  137.            Special_token,
  138.            Input_list_of_chars,
  139.            Left_over_tail_of_input)
  140.  
  141. Special_token is the name of a 3-arg. predicate such as is defined by
  142. DCG rules.  It returns special tokens.
  143.  
  144. For example, we might define
  145. a DCG rule to identify maker strings like this: ****************.
  146. The definition of a DCG to do this is
  147.  
  148. */
  149.  
  150. marker_string( marker(Token ) )  --> [`*], !, marker_tail(T),
  151.                                      {list_text( [`* |T], Token)}.
  152.  
  153. marker_tail([`*|T])  --> [`*], !, marker_tail(T).
  154. marker_tail( []) --> !.
  155.  
  156. /*
  157.  
  158. Calling
  159.  
  160. list_text( Input_list_of_chars, $123.abc*****$),
  161. tokens(Token,
  162.        marker,
  163.        Input_list_of_chars,
  164.        Left_over_tail_of_input)
  165.  
  166. instantiates
  167.  
  168. Token = [ digits($123$), char($.$), alphanum($abc$), marker($*****$)]
  169.  
  170. FILES TO RECONSULT IN ADDITION TO THIS ONE:
  171.  
  172. :- reconsult($\lib\strings\IS_CHAR$).
  173. :- reconsult($\lib\strings\scan\sep$).
  174.  
  175. EXAMPLE:
  176. */
  177.  
  178. testpred :-
  179.    write($Enter a line: $),
  180.    read_line(0,Response),
  181.    list_text(List, Response),
  182.    next_token(Token,
  183.               marker_string,
  184.               List,
  185.               Left_over_tail_of_input),
  186.    write($ Token = $),write(Token),nl,
  187.    write($ Left_over_tail_of_input = $),
  188.    write(Left_over_tail_of_input),nl,
  189.        % now call tokens
  190.    tokens(Tokens,
  191.           marker_string,
  192.           List,
  193.           []),
  194.    write($ Tokens = $),write(Tokens),nl.
  195.  
  196.  
  197. %%%%%%%%%%%%%%%%% code starts here %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  198.  
  199. next_token(Token) -->   next_token(Token, fail ).
  200.  
  201.  
  202. next_token(Structure, Rulepred, Chars, Left_over)  :-
  203.               current_predicate(Rulepred / 3),
  204.               Term =.. [Rulepred, Structure, Chars, Left_over],
  205.               call(Term),!.
  206.  
  207. next_token(Token, Rulepred) -->   separator(_),!,
  208.                                   next_token(Token, Rulepred).
  209.  
  210. next_token(alphanum(Token),_)  -->  alpha(A), !, alphanum_tail(T),
  211.                                     {list_text([A|T],Token)}.
  212.  
  213. next_token(digits(Token),_)  -->  digit(A), !, digit_tail(T),
  214.                                   {list_text([A|T],Token)}.
  215.  
  216.  
  217. next_token(eof($EOF$),_)  -->  [26], !.
  218.  
  219. next_token(char(Token),_)  -->  [T], !,
  220.                                 {list_text([T],Token)}, !.
  221.  
  222. alphanum_tail([H|T])  --> alphanum(H),
  223.                           alphanum_tail(T),!.
  224. alphanum_tail([]) -->!.
  225.  
  226. digit_tail([H|T])  --> digit(H),
  227.                           digit_tail(T),!.
  228. digit_tail([]) -->!.
  229.  
  230.  
  231. digit(A) --> [A], {is_digit(A)}.
  232.  
  233. alpha(A) --> [A], {is_letter(A)}.
  234.  
  235. alphanum(H) --> [H],{is_alphanum( H ) }.
  236.  
  237. %%%%%%%%%%%%%%%%%%%%%%%%% tokens %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  238. /* finds all the tokens in a list of chars */
  239.  
  240. tokens(L) --> tokens(L, fail).
  241.  
  242. tokens([H|T], Rulepred) --> next_token( H, Rulepred ), !,
  243.                           tokens(T, Rulepred).
  244. tokens([], _ ) -->  !.
  245.  
  246.  
  247.  
  248. %%%%%%%%%%%%%%%% end next_token %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  249.