home *** CD-ROM | disk | FTP | other *** search
/ Enter 2005 March / ENTER.ISO / files / fwp-0.0.6-win32-installer.exe / Tokenizer.cpp < prev    next >
Encoding:
C/C++ Source or Header  |  2004-02-03  |  2.7 KB  |  113 lines
  1. #include "Tokenizer.h"
  2.  
  3. #include <string.h>
  4.  
  5. Tokenizer::Tokenizer(const char* string, const char* separators, const char* brackets){
  6.     for(int i=0;i<TOKENIZER_MAX_TOKENS;i++)
  7.         tokv[i]=NULL;
  8.  
  9.     this->separators=separators;
  10.     this->brackets=brackets;
  11.  
  12.     tokenize(string);
  13. }
  14.  
  15. Tokenizer::Tokenizer(const char* separators, const char* brackets){
  16.     for(int i=0;i<TOKENIZER_MAX_TOKENS;i++)
  17.         tokv[i]=NULL;
  18.  
  19.     this->separators=separators;
  20.     this->brackets=brackets;
  21. }
  22.  
  23. Tokenizer::~Tokenizer(){
  24.     clearTokens();        // free allocated memory for tokv
  25. }
  26.  
  27.  
  28. bool Tokenizer::tokenize(const char* string){
  29.     char* p=(char*)string;
  30.     char* startp=(char*)string;
  31.     bool bracketsOpened=false;
  32.  
  33.     clearTokens();
  34.     
  35.     while(*p!='\0'){
  36.         if(contains(separators, *p) && !bracketsOpened){        // if we've read a 'bracket'-char all seps are ignored until
  37.             if(startp!=p){                                        // we reach the 'closing bracket'.
  38.                 if(!addToken(startp, p-startp))
  39.                     return false;
  40.             }
  41.             startp=p+1;
  42.         }
  43.  
  44.         if(contains(brackets, *p)){
  45.             if(bracketsOpened){                        // bracket opened -> close bracket
  46.                 if(contains(separators, *p)){        // if bracket is also separator add token
  47.                     if(!addToken(startp, p-startp))
  48.                         return false;
  49.                     startp=p+1;
  50.                 }
  51.                 bracketsOpened=false;
  52.             }else{                                    // opening bracket -> just remember that in 'bracketsOpened'
  53. //                if(startp!=p){
  54. //                    if(!addToken(startp, p-startp))
  55. //                        return false;
  56. //                }
  57.                 bracketsOpened=true;
  58.             }
  59. //            startp=p+1;        // ignore 'brackets'
  60.         }
  61.         p++;
  62.         if(*p=='\0' && startp!=p){                // make sure we also get the last token in the string
  63.             addToken(startp, p-startp);
  64.         }
  65.     }
  66.  
  67.  
  68.     return true;
  69. }
  70.  
  71.  
  72. bool Tokenizer::contains(const char* str, char c){
  73.     char *p=(char*)str;
  74.  
  75.     while(*p!='\0'){        // go through entire string and see, if we can find character 'c'
  76.         if(c==*p)
  77.             return true;
  78.         p++;
  79.     }
  80.     return false;
  81. }
  82.  
  83. bool Tokenizer::addToken(char *start, unsigned int length){
  84.     if(tokc>=TOKENIZER_MAX_TOKENS)
  85.         return false;        // no more free slots
  86.  
  87.     tokv[tokc]=strncpy(new char[length+1], start, length);    // allocate mem and copy token into it
  88.     tokv[tokc][length]='\0';                                // append \0
  89.     tokc++;                                                    // and finaly increase counter
  90.     return true;
  91. }
  92.  
  93. void Tokenizer::clearTokens(){
  94.     tokc=0;                                        // reset token counter
  95.     for(int i=0; i<TOKENIZER_MAX_TOKENS; i++){    // delete all tokens from previous calls
  96.         if(tokv[i]!=NULL){
  97.             delete[] tokv[i];
  98.             tokv[i]=NULL;
  99.         }
  100.     }
  101. }
  102.  
  103. char* newString(const char* str){
  104.     return strcpy(new char[strlen(str)+1], (str));
  105. }
  106.  
  107. char* newString(const char* start, int length){
  108.     char* ret=strncpy(new char[length+1], start, length);
  109.     ret[length]='\0';
  110.  
  111.     return ret;
  112. }
  113.