home *** CD-ROM | disk | FTP | other *** search
/ MacFormat 1995 June / MacFormat 25.iso / Shareware City / Developers / OutOfPhase1.1 Source / OutOfPhase Folder / CompilerScanner.c < prev    next >
Encoding:
C/C++ Source or Header  |  1994-12-27  |  23.0 KB  |  898 lines  |  [TEXT/KAHL]
  1. /* CompilerScanner.c */
  2. /*****************************************************************************/
  3. /*                                                                           */
  4. /*    Out Of Phase:  Digital Music Synthesis on General Purpose Computers    */
  5. /*    Copyright (C) 1994  Thomas R. Lawrence                                 */
  6. /*                                                                           */
  7. /*    This program is free software; you can redistribute it and/or modify   */
  8. /*    it under the terms of the GNU General Public License as published by   */
  9. /*    the Free Software Foundation; either version 2 of the License, or      */
  10. /*    (at your option) any later version.                                    */
  11. /*                                                                           */
  12. /*    This program is distributed in the hope that it will be useful,        */
  13. /*    but WITHOUT ANY WARRANTY; without even the implied warranty of         */
  14. /*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the          */
  15. /*    GNU General Public License for more details.                           */
  16. /*                                                                           */
  17. /*    You should have received a copy of the GNU General Public License      */
  18. /*    along with this program; if not, write to the Free Software            */
  19. /*    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.              */
  20. /*                                                                           */
  21. /*    Thomas R. Lawrence can be reached at tomlaw@world.std.com.             */
  22. /*                                                                           */
  23. /*****************************************************************************/
  24.  
  25. #include "MiscInfo.h"
  26. #include "Audit.h"
  27. #include "Debug.h"
  28. #include "Definitions.h"
  29.  
  30. #include "CompilerScanner.h"
  31. #include "TrashTracker.h"
  32. #include "Memory.h"
  33. #include "DataMunging.h"
  34. #include "StringThing.h"
  35. #include "Numbers.h"
  36.  
  37.  
  38. #define MAXNUMKEYWORDS (128)
  39.  
  40.  
  41. #define OPAREN '('
  42. #define CPAREN ')'
  43. #define OBRACKET '['
  44. #define CBRACKET ']'
  45.  
  46.  
  47. struct TokenRec
  48.     {
  49.         TokenTypes                Type;
  50.         union
  51.             {
  52.                 long                            IntegerValue;
  53.                 double                        DoubleValue;
  54.                 float                            SingleValue;
  55.                 MyBoolean                    BooleanValue;
  56.                 char*                            StringValue;
  57.                 largefixedsigned    FixedValue;
  58.             } u;
  59.     };
  60.  
  61.  
  62. typedef struct
  63.     {
  64.         char*                            KeywordName;
  65.         long                            KeywordLength;
  66.         long                            TagValue;
  67.     } KeywordRec;
  68.  
  69.  
  70. struct ScannerRec
  71.     {
  72.         char*                            Block;
  73.         long                            BlockLength;
  74.         long                            Index;
  75.         long                            LineNumber;
  76.         long                            NumKeywords;
  77.         TrashTrackRec*        Allocator;
  78.         TokenRec*                    PushedBackToken;
  79.         KeywordRec                KeywordList[MAXNUMKEYWORDS];
  80.     };
  81.  
  82.  
  83. /* this type is used when parsing floating point numbers to remember what */
  84. /* part we're on */
  85. typedef enum
  86.     {
  87.         eIntegerPart EXECUTE(= -6751),
  88.         eFractionalPart,
  89.         eExponentialPart,
  90.         eExponNumberPart,
  91.         eNumberFinished
  92.     } NumStateType;
  93.  
  94.  
  95. /* this type is used for explicitly specifying the type of a number */
  96. typedef enum
  97.     {
  98.         eTypeNotSpecified EXECUTE(= -5612),
  99.         eTypeSingle,
  100.         eTypeDouble,
  101.         eTypeFixed,
  102.         eTypeInteger
  103.     } NumFormType;
  104.  
  105.  
  106. /* create a scanner information record */
  107. ScannerRec*                    NewScanner(struct TrashTrackRec* TrashTracker, char* RawData)
  108.     {
  109.         ScannerRec*                Scanner;
  110.  
  111.         CheckPtrExistence(RawData);
  112.  
  113.         Scanner = (ScannerRec*)AllocTrackedBlock(sizeof(ScannerRec),TrashTracker);
  114.         if (Scanner == NIL)
  115.             {
  116.                 return NIL;
  117.             }
  118.         SetTag(Scanner,"ScannerRec");
  119.  
  120.         Scanner->Block = AllocTrackedBlock(PtrSize(RawData),TrashTracker);
  121.         if (Scanner->Block == NIL)
  122.             {
  123.                 return NIL;
  124.             }
  125.         SetTag(Scanner->Block,"ScannerRec:  data block");
  126.  
  127.         CopyData(RawData,Scanner->Block,PtrSize(RawData));
  128.  
  129.         Scanner->BlockLength = PtrSize(Scanner->Block);
  130.         Scanner->Index = 0;
  131.         Scanner->LineNumber = 1;
  132.         Scanner->NumKeywords = 0;
  133.         Scanner->Allocator = TrashTracker;
  134.         Scanner->PushedBackToken = NIL;
  135.  
  136.         return Scanner;
  137.     }
  138.  
  139.  
  140. /* negative = L < R; zero = equal, positive = L > R */
  141. static int                    CompareStrings(char* LRef, long LLen, char* RRef, long RLen)
  142.     {
  143.         long                            Scan;
  144.  
  145.         Scan = 0;
  146.         while ((Scan < LLen) && (Scan < RLen))
  147.             {
  148.                 int                                Result;
  149.  
  150.                 Result = LRef[Scan] - RRef[Scan];
  151.                 if (Result != 0)
  152.                     {
  153.                         /* negative means LRef < RRef */
  154.                         /* positive means LRef > RRef */
  155.                         return Result;
  156.                     }
  157.                 Scan += 1;
  158.             }
  159.         /* if strings are equal up to smallest lengthed one, then */
  160.         /* if left is shorter, then left is less */
  161.         /* if right is shorter, then right is less */
  162.         /* if lengths are equal, then strings are equal */
  163.         return LLen - RLen;
  164.     }
  165.  
  166.  
  167. /* add a token to the scanner.  Keyword is a null terminated statically */
  168. /* allocated string */
  169. void                                AddKeywordToScanner(ScannerRec* Scanner, char* Keyword, long Tag)
  170.     {
  171.         long                            Scan;
  172.         long                            KeywordLength;
  173.  
  174.         CheckPtrExistence(Scanner);
  175.  
  176.         ERROR(Scanner->NumKeywords == MAXNUMKEYWORDS,PRERR(ForceAbort,
  177.             "AddKeywordToScanner:  too many keywords"));
  178.  
  179.         KeywordLength = StrLen(Keyword);
  180.         Scan = 0;
  181.         while (Scan < Scanner->NumKeywords)
  182.             {
  183.                 int                                Compare;
  184.  
  185.                 Compare = CompareStrings(Keyword,KeywordLength,Scanner->KeywordList[Scan]
  186.                     .KeywordName,Scanner->KeywordList[Scan].KeywordLength);
  187.                 ERROR(Compare == 0,PRERR(ForceAbort,
  188.                     "AddKeywordToScanner:  keyword already known"));
  189.                 if (Compare < 0)
  190.                     {
  191.                         long                            Index;
  192.  
  193.                         /* our string is less, so insert at Scan */
  194.                         for (Index = Scanner->NumKeywords; Index > Scan; Index -= 1)
  195.                             {
  196.                                 Scanner->KeywordList[Index] = Scanner->KeywordList[Index - 1];
  197.                             }
  198.                         Scanner->KeywordList[Scan].KeywordName = Keyword;
  199.                         Scanner->KeywordList[Scan].KeywordLength = KeywordLength;
  200.                         Scanner->KeywordList[Scan].TagValue = Tag;
  201.                         Scanner->NumKeywords += 1;
  202.                         return;
  203.                     }
  204.                 /* else go to the next one */
  205.                 Scan += 1;
  206.             }
  207.         /* if there was nowhere to insert, then just put it in there hey */
  208.         Scanner->KeywordList[Scanner->NumKeywords].KeywordName = Keyword;
  209.         Scanner->KeywordList[Scanner->NumKeywords].KeywordLength = KeywordLength;
  210.         Scanner->KeywordList[Scanner->NumKeywords].TagValue = Tag;
  211.         Scanner->NumKeywords += 1;
  212.     }
  213.  
  214.  
  215. /* get the line number (starting at 1) that the scanner is on right now */
  216. long                                GetCurrentLineNumber(ScannerRec* Scanner)
  217.     {
  218.         CheckPtrExistence(Scanner);
  219.         return Scanner->LineNumber;
  220.     }
  221.  
  222.  
  223. #define ENDOFTEXT (-1)
  224.  
  225. static int                    GetCharacter(ScannerRec* Scanner)
  226.     {
  227.         int                                Value;
  228.  
  229.         CheckPtrExistence(Scanner);
  230.         if (Scanner->Index >= Scanner->BlockLength)
  231.             {
  232.                 Scanner->Index += 1; /* increment so that we can unget end of text */
  233.                 return ENDOFTEXT;
  234.             }
  235.         Value = Scanner->Block[Scanner->Index];
  236.         Scanner->Index += 1;
  237.         return Value;
  238.     }
  239.  
  240.  
  241. static void                    UngetCharacter(ScannerRec* Scanner)
  242.     {
  243.         CheckPtrExistence(Scanner);
  244.         ERROR(Scanner->Index == 0,PRERR(ForceAbort,"UngetCharacter:  can't do it"));
  245.         Scanner->Index -= 1;
  246.     }
  247.  
  248.  
  249. /* get a token */
  250. TokenRec*                        GetNextToken(ScannerRec* Scanner)
  251.     {
  252.         int                                C;
  253.         TokenRec*                    Token;
  254.  
  255.         CheckPtrExistence(Scanner);
  256.  
  257.         /* check for pushback */
  258.         if (Scanner->PushedBackToken != NIL)
  259.             {
  260.                 TokenRec*                    Temp;
  261.  
  262.                 Temp = Scanner->PushedBackToken;
  263.                 Scanner->PushedBackToken = NIL;
  264.                 return Temp;
  265.             }
  266.  
  267.         /* get a character */
  268.         C = GetCharacter(Scanner);
  269.  
  270.         /* strip while space */
  271.         while (((C >= 0) && (C <= 32)) || (C == '#'))
  272.             {
  273.                 if ((C == 13) || (C == 10))
  274.                     {
  275.                         Scanner->LineNumber += 1;
  276.                     }
  277.                 if (C == '#')
  278.                     {
  279.                         /* comment */
  280.                         while ((C != 13) && (C != 10) && (C != ENDOFTEXT))
  281.                             {
  282.                                 C = GetCharacter(Scanner);
  283.                             }
  284.                     }
  285.                  else
  286.                     {
  287.                         C = GetCharacter(Scanner);
  288.                     }
  289.             }
  290.  
  291.         /* handle the end of text character */
  292.         if (C == ENDOFTEXT)
  293.             {
  294.                 Token = (TokenRec*)AllocTrackedBlock(sizeof(TokenRec),Scanner->Allocator);
  295.                 if (Token == NIL)
  296.                     {
  297.                         return NIL;
  298.                     }
  299.                 Token->Type = eTokenEndOfInput;
  300.             }
  301.  
  302.         /* handle a string literal */
  303.         else if (C == '\x22')
  304.             {
  305.                 StringThingRec*        String;
  306.                 char*                            StringCopy;
  307.                 long                            StringLength;
  308.  
  309.                 String = NewStringThing();
  310.                 if (String == NIL)
  311.                     {
  312.                         return NIL;
  313.                     }
  314.  
  315.                 C = GetCharacter(Scanner);
  316.                 while (C != '\x22')
  317.                     {
  318.                         char                            Buffer[1];
  319.  
  320.                         if (C == ENDOFTEXT)
  321.                             {
  322.                                 goto BreakStringReadPoint;
  323.                             }
  324.                         if (C == '\\')
  325.                             {
  326.                                 C = GetCharacter(Scanner);
  327.                                 if (C == 'n')
  328.                                     {
  329.                                         if (!AppendStringThing(String,SYSTEMLINEFEED,StrLen(SYSTEMLINEFEED)))
  330.                                             {
  331.                                                 DisposeStringThing(String);
  332.                                                 return NIL;
  333.                                             }
  334.                                         goto DoAnotherCharPoint;
  335.                                     }
  336.                                 else if ((C == '\x22') || (C == '\\') || (C == 10) || (C == 13))
  337.                                     {
  338.                                         /* keep these */
  339.                                     }
  340.                                 else
  341.                                     {
  342.                                         /* others become strange character */
  343.                                         C = '.';
  344.                                     }
  345.                             }
  346.                         Buffer[0] = C;
  347.                         if (!AppendStringThing(String,&(Buffer[0]),1))
  348.                             {
  349.                                 DisposeStringThing(String);
  350.                                 return NIL;
  351.                             }
  352.                         if ((C == 10) || (C == 13))
  353.                             {
  354.                                 Scanner->LineNumber += 1;
  355.                             }
  356.                      DoAnotherCharPoint:
  357.                         C = GetCharacter(Scanner);
  358.                     }
  359.              BreakStringReadPoint:
  360.                 ;
  361.  
  362.                 StringCopy = StringThingGetSubrange(String,0,GetStringThingLength(String));
  363.                 DisposeStringThing(String);
  364.                 if (StringCopy == NIL)
  365.                     {
  366.                         return NIL;
  367.                     }
  368.                 StringLength = PtrSize(StringCopy);
  369.  
  370.                 Token = (TokenRec*)AllocTrackedBlock(sizeof(TokenRec),Scanner->Allocator);
  371.                 if (Token == NIL)
  372.                     {
  373.                         ReleasePtr(StringCopy);
  374.                         return NIL;
  375.                     }
  376.                 Token->Type = eTokenString;
  377.  
  378.                 Token->u.StringValue = AllocTrackedBlock(StringLength,Scanner->Allocator);
  379.                 if (Token->u.StringValue == NIL)
  380.                     {
  381.                         ReleasePtr(StringCopy);
  382.                         return NIL;
  383.                     }
  384.                 CopyData(StringCopy,Token->u.StringValue,StringLength);
  385.                 ReleasePtr(StringCopy);
  386.             }
  387.  
  388.         /* handle an identifier:  [a-zA-Z_][a-zA-Z0-9_]*  */
  389.         else if (((C >= 'a') && (C <= 'z')) || ((C >= 'A') && (C <= 'Z')) || (C == '_'))
  390.             {
  391.                 StringThingRec*        String;
  392.                 long                            LowBound;
  393.                 long                            HighBoundPlusOne;
  394.                 MyBoolean                    ContinueLoopingFlag;
  395.                 long                            KeywordIndex; /* -1 == not a keyword */
  396.                 char*                            StringCopy;
  397.                 long                            StringLength;
  398.  
  399.                 String = NewStringThing();
  400.                 if (String == NIL)
  401.                     {
  402.                         return NIL;
  403.                     }
  404.                 /* read the entire token */
  405.                 while (((C >= 'a') && (C <= 'z')) || ((C >= 'A') && (C <= 'Z')) || (C == '_')
  406.                     || ((C >= '0') && (C <= '9')))
  407.                     {
  408.                         char                            Buffer[1];
  409.  
  410.                         Buffer[0] = C;
  411.                         if (!AppendStringThing(String,&(Buffer[0]),1))
  412.                             {
  413.                                 DisposeStringThing(String);
  414.                                 return NIL;
  415.                             }
  416.                         C = GetCharacter(Scanner);
  417.                     }
  418.                 /* unget the character that made us stop */
  419.                 UngetCharacter(Scanner);
  420.                 /* get the string out of the line buffer */
  421.                 StringCopy = StringThingGetSubrange(String,0,GetStringThingLength(String));
  422.                 DisposeStringThing(String);
  423.                 if (StringCopy == NIL)
  424.                     {
  425.                         return NIL;
  426.                     }
  427.                 StringLength = PtrSize(StringCopy);
  428.  
  429.                 /* figure out if it is a keyword */
  430.                 LowBound = 0;
  431.                 HighBoundPlusOne = Scanner->NumKeywords;
  432.                 ContinueLoopingFlag = True;
  433.                 while (ContinueLoopingFlag)
  434.                     {
  435.                         long                            MidPoint;
  436.                         int                                CompareResult;
  437.  
  438.                         ERROR(LowBound > HighBoundPlusOne,PRERR(ForceAbort,
  439.                             "GetNextToken:  we seem to have a problem with low and high bounds"));
  440.  
  441.                         MidPoint = (LowBound + HighBoundPlusOne) / 2;
  442.  
  443.                         CompareResult = CompareStrings(StringCopy,PtrSize(StringCopy),
  444.                             Scanner->KeywordList[MidPoint].KeywordName,
  445.                             Scanner->KeywordList[MidPoint].KeywordLength);
  446.                         /* CompareResult == 0  -->  found the target */
  447.                         /* CompareResult < 0  --> in the first half of the list */
  448.                         /* CompareResult > 0  --> in the second half of the list */
  449.  
  450.                         if (CompareResult == 0)
  451.                             {
  452.                                 /* found the one */
  453.                                 KeywordIndex = MidPoint;
  454.                                 ContinueLoopingFlag = False;
  455.                             }
  456.                          else
  457.                             {
  458.                                 if (CompareResult < 0)
  459.                                     {
  460.                                         /* select first half of list */
  461.                                         HighBoundPlusOne = MidPoint;
  462.                                     }
  463.                                 else /* if (CompareResult > 0) */
  464.                                     {
  465.                                         /* select second half of list */
  466.                                         LowBound = MidPoint + 1;
  467.                                     }
  468.                                 /* termination condition:  if range in array collapses to an */
  469.                                 /* empty array, then there is no entry in the array */
  470.                                 if (LowBound == HighBoundPlusOne)
  471.                                     {
  472.                                         KeywordIndex = -1; /* indicate there is no keyword */
  473.                                         ContinueLoopingFlag = False;
  474.                                     }
  475.                             }
  476.                     }
  477.  
  478.                 /* create the token */
  479.                 Token = (TokenRec*)AllocTrackedBlock(sizeof(TokenRec),Scanner->Allocator);
  480.                 if (Token == NIL)
  481.                     {
  482.                         ReleasePtr(StringCopy);
  483.                         return NIL;
  484.                     }
  485.  
  486.                 if (KeywordIndex == -1)
  487.                     {
  488.                         /* no keyword; make a string containing token */
  489.                         Token->u.StringValue = AllocTrackedBlock(StringLength,Scanner->Allocator);
  490.                         if (Token->u.StringValue == NIL)
  491.                             {
  492.                                 ReleasePtr(StringCopy);
  493.                                 return NIL;
  494.                             }
  495.                         CopyData(StringCopy,Token->u.StringValue,StringLength);
  496.                         Token->Type = eTokenIdentifier;
  497.                     }
  498.                  else
  499.                     {
  500.                         Token->Type = eTokenKeyword;
  501.                         Token->u.IntegerValue = Scanner->KeywordList[KeywordIndex].TagValue;
  502.                     }
  503.                 ReleasePtr(StringCopy);
  504.             }
  505.  
  506.         /* integer or floating?  [0-9]+  [0-9]+"."[0-9]+([Ee][+-]?[0-9]+)?[sdf]?  */
  507.         else if (((C >= '0') && (C <= '9')) || (C == '.'))
  508.             {
  509.                 NumFormType                SpecifiedNumType;
  510.                 NumStateType            NumberState;
  511.                 StringThingRec*        String;
  512.                 char*                            StringData;
  513.                 long                            StringLength;
  514.  
  515.                 /* initialize the state value */
  516.                 NumberState = eIntegerPart;
  517.                 SpecifiedNumType = eTypeNotSpecified;
  518.  
  519.                 Token = (TokenRec*)AllocTrackedBlock(sizeof(TokenRec),Scanner->Allocator);
  520.                 if (Token == NIL)
  521.                     {
  522.                         return NIL;
  523.                     }
  524.  
  525.                 String = NewStringThing();
  526.                 if (String == NIL)
  527.                     {
  528.                         return NIL;
  529.                     }
  530.  
  531.                 while (((C >= '0') && (C <= '9')) || (C == '.') || (C == '+') || (C == '-')
  532.                     || (C == 's') || (C == 'd') || (C == 'f') || (C == 'e') || (C == 'E'))
  533.                     {
  534.                         char                            Buffer[1];
  535.  
  536.                         /* do some state changes */
  537.                         if (C == '.')
  538.                             {
  539.                                 if (NumberState != eIntegerPart)
  540.                                     {
  541.                                         Token->Type = eTokenError;
  542.                                         Token->u.IntegerValue = eScannerMalformedFloat;
  543.                                         DisposeStringThing(String);
  544.                                         goto AbortNumberErrorPoint;
  545.                                     }
  546.                                  else
  547.                                     {
  548.                                         NumberState = eFractionalPart;
  549.                                     }
  550.                             }
  551.                         else if ((C == 'e') || (C == 'E'))
  552.                             {
  553.                                 if ((NumberState != eIntegerPart) && (NumberState != eFractionalPart))
  554.                                     {
  555.                                         Token->Type = eTokenError;
  556.                                         Token->u.IntegerValue = eScannerMalformedFloat;
  557.                                         DisposeStringThing(String);
  558.                                         goto AbortNumberErrorPoint;
  559.                                     }
  560.                                  else
  561.                                     {
  562.                                         NumberState = eExponentialPart;
  563.                                     }
  564.                             }
  565.                         else if ((C == '+') || (C == '-'))
  566.                             {
  567.                                 if (NumberState != eExponentialPart)
  568.                                     {
  569.                                         /* this is not an error, since it could be a unary operator */
  570.                                         /* coming later, so we stop, but don't abort */
  571.                                         goto FinishNumberPoint; /* character ungot at target */
  572.                                     }
  573.                                  else
  574.                                     {
  575.                                         NumberState = eExponNumberPart;
  576.                                     }
  577.                             }
  578.                         else if (C == 's')
  579.                             {
  580.                                 if (NumberState == eNumberFinished)
  581.                                     {
  582.                                         Token->Type = eTokenError;
  583.                                         Token->u.IntegerValue = eScannerMalformedFloat;
  584.                                         DisposeStringThing(String);
  585.                                         goto AbortNumberErrorPoint;
  586.                                     }
  587.                                  else
  588.                                     {
  589.                                         NumberState = eNumberFinished;
  590.                                         SpecifiedNumType = eTypeSingle;
  591.                                         C = 32; /* so adding it to the string doesn't do damage */
  592.                                     }
  593.                             }
  594.                         else if (C == 'd')
  595.                             {
  596.                                 if (NumberState == eNumberFinished)
  597.                                     {
  598.                                         Token->Type = eTokenError;
  599.                                         Token->u.IntegerValue = eScannerMalformedFloat;
  600.                                         DisposeStringThing(String);
  601.                                         goto AbortNumberErrorPoint;
  602.                                     }
  603.                                  else
  604.                                     {
  605.                                         NumberState = eNumberFinished;
  606.                                         SpecifiedNumType = eTypeDouble;
  607.                                         C = 32;
  608.                                     }
  609.                             }
  610.                         else if (C == 'f')
  611.                             {
  612.                                 if (NumberState == eNumberFinished)
  613.                                     {
  614.                                         Token->Type = eTokenError;
  615.                                         Token->u.IntegerValue = eScannerMalformedFloat;
  616.                                         DisposeStringThing(String);
  617.                                         goto AbortNumberErrorPoint;
  618.                                     }
  619.                                  else
  620.                                     {
  621.                                         NumberState = eNumberFinished;
  622.                                         SpecifiedNumType = eTypeFixed;
  623.                                         C = 32;
  624.                                     }
  625.                             }
  626.  
  627.                         /* actually save the character */
  628.                         Buffer[0] = C;
  629.                         if (!AppendStringThing(String,&(Buffer[0]),1))
  630.                             {
  631.                                 DisposeStringThing(String);
  632.                                 return NIL;
  633.                             }
  634.  
  635.                         C = GetCharacter(Scanner);
  636.                     }
  637.              FinishNumberPoint:
  638.                 UngetCharacter(Scanner);
  639.  
  640.                 StringData = StringThingGetSubrange(String,0,GetStringThingLength(String));
  641.                 DisposeStringThing(String);
  642.                 if (StringData == NIL)
  643.                     {
  644.                         return NIL;
  645.                     }
  646.                 StringLength = PtrSize(StringData);
  647.  
  648.                 /* if the token type is not specified, then see what we can guess */
  649.                 if (eTypeNotSpecified == SpecifiedNumType)
  650.                     {
  651.                         if (NumberState == eIntegerPart)
  652.                             {
  653.                                 /* if we only got as far as the integer part, then it's an int */
  654.                                 SpecifiedNumType = eTypeInteger;
  655.                             }
  656.                          else
  657.                             {
  658.                                 /* otherwise, assume the highest precision type */
  659.                                 SpecifiedNumType = eTypeDouble;
  660.                             }
  661.                     }
  662.  
  663.                 /* create the token */
  664.                 switch (SpecifiedNumType)
  665.                     {
  666.                         default:
  667.                             EXECUTE(PRERR(ForceAbort,"GetNextToken:  bad number type specifier"));
  668.                             break;
  669.                         case eTypeSingle:
  670.                             Token->Type = eTokenSingle;
  671.                             Token->u.SingleValue = StringToLongDouble(StringData,StringLength);
  672.                             break;
  673.                         case eTypeDouble:
  674.                             Token->Type = eTokenDouble;
  675.                             Token->u.DoubleValue = StringToLongDouble(StringData,StringLength);
  676.                             break;
  677.                         case eTypeFixed:
  678.                             Token->Type = eTokenFixed;
  679.                             Token->u.FixedValue = double2largefixed(StringToLongDouble(StringData,
  680.                                 StringLength));
  681.                             break;
  682.                         case eTypeInteger:
  683.                             Token->Type = eTokenInteger;
  684.                             Token->u.IntegerValue = StringToInteger(StringData,StringLength);
  685.                             break;
  686.                     }
  687.                 ReleasePtr(StringData);
  688.  
  689.                 /* this is the escape point for when a bad character is encountered. */
  690.              AbortNumberErrorPoint:
  691.                 ;
  692.             }
  693.  
  694.         /* handle a symbol */
  695.         else
  696.             {
  697.                 Token = (TokenRec*)AllocTrackedBlock(sizeof(TokenRec),Scanner->Allocator);
  698.                 if (Token == NIL)
  699.                     {
  700.                         return NIL;
  701.                     }
  702.  
  703.                 switch (C)
  704.                     {
  705.                         default:
  706.                             Token->Type = eTokenError;
  707.                             Token->u.IntegerValue = eScannerUnknownCharacter;
  708.                             break;
  709.                         case OPAREN:
  710.                             Token->Type = eTokenOpenParen;
  711.                             break;
  712.                         case CPAREN:
  713.                             Token->Type = eTokenCloseParen;
  714.                             break;
  715.                         case OBRACKET:
  716.                             Token->Type = eTokenOpenBracket;
  717.                             break;
  718.                         case CBRACKET:
  719.                             Token->Type = eTokenCloseBracket;
  720.                             break;
  721.                         case ':':
  722.                             C = GetCharacter(Scanner);
  723.                             if (C == '=')
  724.                                 {
  725.                                     Token->Type = eTokenColonEqual;
  726.                                 }
  727.                              else
  728.                                 {
  729.                                     /* push the character back */
  730.                                     UngetCharacter(Scanner);
  731.                                     Token->Type = eTokenColon;
  732.                                 }
  733.                             break;
  734.                         case ';':
  735.                             Token->Type = eTokenSemicolon;
  736.                             break;
  737.                         case ',':
  738.                             Token->Type = eTokenComma;
  739.                             break;
  740.                         case '+':
  741.                             Token->Type = eTokenPlus;
  742.                             break;
  743.                         case '-':
  744.                             Token->Type = eTokenMinus;
  745.                             break;
  746.                         case '*':
  747.                             Token->Type = eTokenStar;
  748.                             break;
  749.                         case '/':
  750.                             Token->Type = eTokenSlash;
  751.                             break;
  752.                         case '=':
  753.                             Token->Type = eTokenEqual;
  754.                             break;
  755.                         case '<':
  756.                             C = GetCharacter(Scanner);
  757.                             if (C == '>')
  758.                                 {
  759.                                     Token->Type = eTokenNotEqual;
  760.                                 }
  761.                             else if (C == '=')
  762.                                 {
  763.                                     Token->Type = eTokenLessThanOrEqual;
  764.                                 }
  765.                             else if (C == '<')
  766.                                 {
  767.                                     Token->Type = eTokenShiftLeft;
  768.                                 }
  769.                             else
  770.                                 {
  771.                                     Token->Type = eTokenLessThan;
  772.                                     UngetCharacter(Scanner);
  773.                                 }
  774.                             break;
  775.                         case '>':
  776.                             C = GetCharacter(Scanner);
  777.                             if (C == '=')
  778.                                 {
  779.                                     Token->Type = eTokenGreaterThanOrEqual;
  780.                                 }
  781.                             else if (C == '>')
  782.                                 {
  783.                                     Token->Type = eTokenShiftRight;
  784.                                 }
  785.                             else
  786.                                 {
  787.                                     Token->Type = eTokenGreaterThan;
  788.                                     UngetCharacter(Scanner);
  789.                                 }
  790.                             break;
  791.                         case '^':
  792.                             Token->Type = eTokenCircumflex;
  793.                             break;
  794.                     }
  795.             }
  796.  
  797.         return Token;
  798.     }
  799.  
  800.  
  801. /* push a token back onto the token stream */
  802. void                                UngetToken(ScannerRec* Scanner, TokenRec* Token)
  803.     {
  804.         CheckPtrExistence(Scanner);
  805.         CheckPtrExistence(Token);
  806.         ERROR(Scanner->PushedBackToken != NIL,PRERR(ForceAbort,
  807.             "UngetToken:  there is already a token pushed back"));
  808.         Scanner->PushedBackToken = Token;
  809.     }
  810.  
  811.  
  812. /* get the type of a token */
  813. TokenTypes                    GetTokenType(TokenRec* Token)
  814.     {
  815.         CheckPtrExistence(Token);
  816.         return Token->Type;
  817.     }
  818.  
  819.  
  820. /* get the string associated with an identifier */
  821. char*                                GetTokenIdentifierString(TokenRec* Token)
  822.     {
  823.         CheckPtrExistence(Token);
  824.         ERROR(Token->Type != eTokenIdentifier,PRERR(ForceAbort,
  825.             "GetTokenIdentifierString:  token isn't an identifier"));
  826.         return Token->u.StringValue;
  827.     }
  828.  
  829.  
  830. /* get the string associated with a string token */
  831. char*                                GetTokenStringValue(TokenRec* Token)
  832.     {
  833.         CheckPtrExistence(Token);
  834.         ERROR(Token->Type != eTokenString,PRERR(ForceAbort,
  835.             "GetTokenStringValue:  token isn't a string"));
  836.         return Token->u.StringValue;
  837.     }
  838.  
  839.  
  840. /* get the integer value associated with an integer token */
  841. long                                GetTokenIntegerValue(TokenRec* Token)
  842.     {
  843.         CheckPtrExistence(Token);
  844.         ERROR(Token->Type != eTokenInteger,PRERR(ForceAbort,
  845.             "GetTokenIntegerValue:  token isn't an integer"));
  846.         return Token->u.IntegerValue;
  847.     }
  848.  
  849.  
  850. /* get the single precision value associated with a single float token */
  851. float                                GetTokenSingleValue(TokenRec* Token)
  852.     {
  853.         CheckPtrExistence(Token);
  854.         ERROR(Token->Type != eTokenSingle,PRERR(ForceAbort,
  855.             "GetTokenSingleValue:  token isn't a single"));
  856.         return Token->u.SingleValue;
  857.     }
  858.  
  859.  
  860. /* get the double precision value associated with a double float token */
  861. double                            GetTokenDoubleValue(TokenRec* Token)
  862.     {
  863.         CheckPtrExistence(Token);
  864.         ERROR(Token->Type != eTokenDouble,PRERR(ForceAbort,
  865.             "GetTokenDoubleValue:  token isn't a double"));
  866.         return Token->u.DoubleValue;
  867.     }
  868.  
  869.  
  870. /* get the fixed precision value associated with a fixed token */
  871. largefixedsigned        GetTokenFixedValue(TokenRec* Token)
  872.     {
  873.         CheckPtrExistence(Token);
  874.         ERROR(Token->Type != eTokenFixed,PRERR(ForceAbort,
  875.             "GetTokenFixedValue:  token isn't a fixed point"));
  876.         return Token->u.FixedValue;
  877.     }
  878.  
  879.  
  880. /* get the tag associated with a keyword token */
  881. long                                GetTokenKeywordTag(TokenRec* Token)
  882.     {
  883.         CheckPtrExistence(Token);
  884.         ERROR(Token->Type != eTokenKeyword,PRERR(ForceAbort,
  885.             "GetTokenKeywordTag:  token isn't a keyword"));
  886.         return Token->u.IntegerValue;
  887.     }
  888.  
  889.  
  890. /* get the error code associated with an error token */
  891. ScannerErrors                GetTokenErrorCode(TokenRec* Token)
  892.     {
  893.         CheckPtrExistence(Token);
  894.         ERROR(Token->Type != eTokenError,PRERR(ForceAbort,
  895.             "GetTokenErrorCode:  token isn't a keyword"));
  896.         return (ScannerErrors)Token->u.IntegerValue;
  897.     }
  898.