token.c

   1 \r
   2 //**************************************************************************\r
   3 //**\r
   4 //** token.c\r
   5 //**\r
   6 //**************************************************************************\r
   7 \r
   8 // HEADER FILES ------------------------------------------------------------\r
   9 \r
  10 #ifdef __NeXT__\r
  11 #include <libc.h>\r
  12 #else\r
  13 #ifndef unix\r
  14 #include <io.h>\r
  15 #endif\r
  16 #include <fcntl.h>\r
  17 #include <stdlib.h>\r
  18 #endif\r
  19 #include <stdio.h>\r
  20 #include <string.h>\r
  21 #include <ctype.h>\r
  22 #include "common.h"\r
  23 #include "token.h"\r
  24 #include "error.h"\r
  25 #include "misc.h"\r
  26 #include "symbol.h"\r
  27 #include "parse.h"\r
  28 \r
  29 // MACROS ------------------------------------------------------------------\r
  30 \r
  31 #define NON_HEX_DIGIT 255\r
  32 #define MAX_NESTED_SOURCES 16\r
  33 \r
  34 // TYPES -------------------------------------------------------------------\r
  35 \r
  36 typedef enum\r
  37 {\r
  38         CHR_EOF,\r
  39         CHR_LETTER,\r
  40         CHR_NUMBER,\r
  41         CHR_QUOTE,\r
  42         CHR_SPECIAL\r
  43 } chr_t;\r
  44 \r
  45 typedef struct\r
  46 {\r
  47         char *name;\r
  48         char *start;\r
  49         char *end;\r
  50         char *position;\r
  51         int line;\r
  52         boolean incLineNumber;\r
  53         boolean imported;\r
  54         enum ImportModes prevMode;\r
  55         char lastChar;\r
  56 } nestInfo_t;\r
  57 \r
  58 // EXTERNAL FUNCTION PROTOTYPES --------------------------------------------\r
  59 \r
  60 // PUBLIC FUNCTION PROTOTYPES ----------------------------------------------\r
  61 \r
  62 // PRIVATE FUNCTION PROTOTYPES ---------------------------------------------\r
  63 \r
  64 static int SortKeywords(const void *a, const void *b);\r
  65 static void MakeIncludePath(char *sourceName);\r
  66 static int PopNestedSource(enum ImportModes *prevMode);\r
  67 static void ProcessLetterToken(void);\r
  68 static void ProcessNumberToken(void);\r
  69 static void EvalFixedConstant(int whole);\r
  70 static void EvalHexConstant(void);\r
  71 static void EvalRadixConstant(void);\r
  72 static int DigitValue(char digit, int radix);\r
  73 static void ProcessQuoteToken(void);\r
  74 static void ProcessSpecialToken(void);\r
  75 static boolean CheckForKeyword(void);\r
  76 static boolean CheckForLineSpecial(void);\r
  77 static boolean CheckForConstant(void);\r
  78 static void NextChr(void);\r
  79 static void SkipComment(void);\r
  80 static void SkipCPPComment(void);\r
  81 static void BumpMasterSourceLine(char Chr, boolean clear); // master line - Ty 07jan2000\r
  82 static char *AddFileName(const char *name);\r
  83 static int OctalChar();\r
  84 \r
  85 // EXTERNAL DATA DECLARATIONS ----------------------------------------------\r
  86 \r
  87 // PUBLIC DATA DEFINITIONS -------------------------------------------------\r
  88 \r
  89 tokenType_t tk_Token;\r
  90 int tk_Line;\r
  91 int tk_Number;\r
  92 char *tk_String;\r
  93 U_BYTE tk_SpecialValue;\r
  94 int tk_SpecialArgCount;\r
  95 char *tk_SourceName;\r
  96 int tk_IncludedLines;\r
  97 boolean forSemicolonHack;\r
  98 char MasterSourceLine[MAX_STATEMENT_LENGTH+1];  // master line - Ty 07jan2000\r
  99 int MasterSourcePos;      // master position - Ty 07jan2000\r
 100 int PrevMasterSourcePos;        // previous master position - RH 09feb2000\r
 101 boolean ClearMasterSourceLine;  // master clear flag - Ty 07jan2000\r
 102 \r
 103 // PRIVATE DATA DEFINITIONS ------------------------------------------------\r
 104 \r
 105 static char Chr;\r
 106 static char *FileStart;\r
 107 static char *FilePtr;\r
 108 static char *FileEnd;\r
 109 static boolean SourceOpen;\r
 110 static char ASCIIToChrCode[256];\r
 111 static byte ASCIIToHexDigit[256];\r
 112 static char TokenStringBuffer[MAX_QUOTED_LENGTH];\r
 113 static nestInfo_t OpenFiles[MAX_NESTED_SOURCES];\r
 114 static boolean AlreadyGot;\r
 115 static int NestDepth;\r
 116 static boolean IncLineNumber;\r
 117 static char IncludePath[MAX_FILE_NAME_LENGTH];\r
 118 static char *FileNames;\r
 119 static size_t FileNamesLen, FileNamesMax;\r
 120 \r
 121 static struct keyword_s\r
 122 {\r
 123         char *name;\r
 124         tokenType_t token;\r
 125 } Keywords[] =\r
 126 {\r
 127         { "break", TK_BREAK },\r
 128         { "case", TK_CASE },\r
 129         { "const", TK_CONST },\r
 130         { "continue", TK_CONTINUE },\r
 131         { "default", TK_DEFAULT },\r
 132         { "define", TK_DEFINE },\r
 133         { "do", TK_DO },\r
 134         { "else", TK_ELSE },\r
 135         { "for", TK_FOR },\r
 136         { "goto", TK_GOTO },\r
 137         { "if", TK_IF },\r
 138         { "include", TK_INCLUDE },\r
 139         { "int", TK_INT },\r
 140         { "open", TK_OPEN },\r
 141         { "print", TK_PRINT },\r
 142         { "printbold", TK_PRINTBOLD },\r
 143         { "log", TK_LOG },\r
 144         { "hudmessage", TK_HUDMESSAGE },\r
 145         { "hudmessagebold", TK_HUDMESSAGEBOLD },\r
 146         { "restart", TK_RESTART },\r
 147         { "script", TK_SCRIPT },\r
 148         { "special", TK_SPECIAL },\r
 149         { "str", TK_STR },\r
 150         { "suspend", TK_SUSPEND },\r
 151         { "switch", TK_SWITCH },\r
 152         { "terminate", TK_TERMINATE },\r
 153         { "until", TK_UNTIL },\r
 154         { "void", TK_VOID },\r
 155         { "while", TK_WHILE },\r
 156         { "world", TK_WORLD },\r
 157         { "global", TK_GLOBAL },\r
 158         // [BC] Start Skulltag tokens.\r
 159         { "respawn", TK_RESPAWN },\r
 160         { "death", TK_DEATH },\r
 161         { "enter", TK_ENTER },\r
 162         { "pickup", TK_PICKUP },\r
 163         { "bluereturn", TK_BLUERETURN },\r
 164         { "redreturn", TK_REDRETURN },\r
 165         { "whitereturn", TK_WHITERETURN },\r
 166         // [BC] End Skulltag tokens.\r
 167         { "nocompact", TK_NOCOMPACT },\r
 168         { "lightning", TK_LIGHTNING },\r
 169         { "createtranslation", TK_CREATETRANSLATION },\r
 170         { "function", TK_FUNCTION },\r
 171         { "return", TK_RETURN },\r
 172         { "wadauthor", TK_WADAUTHOR },\r
 173         { "nowadauthor", TK_NOWADAUTHOR },\r
 174         { "acs_executewait", TK_ACSEXECUTEWAIT },\r
 175         { "encryptstrings", TK_ENCRYPTSTRINGS },\r
 176         { "import", TK_IMPORT },\r
 177         { "library", TK_LIBRARY },\r
 178         { "libdefine", TK_LIBDEFINE },\r
 179         { "bool", TK_BOOL },\r
 180         { "net", TK_NET },\r
 181         { "disconnect", TK_DISCONNECT },\r
 182         { "unloading", TK_UNLOADING },\r
 183         { "static", TK_STATIC }\r
 184 };\r
 185 \r
 186 #define NUM_KEYWORDS (sizeof(Keywords)/sizeof(Keywords[0]))\r
 187 \r
 188 // CODE --------------------------------------------------------------------\r
 189 \r
 190 //==========================================================================\r
 191 //\r
 192 // TK_Init\r
 193 //\r
 194 //==========================================================================\r
 195 \r
 196 void TK_Init(void)\r
 197 {\r
 198         int i;\r
 199 \r
 200         for(i = 0; i < 256; i++)\r
 201         {\r
 202                 ASCIIToChrCode[i] = CHR_SPECIAL;\r
 203                 ASCIIToHexDigit[i] = NON_HEX_DIGIT;\r
 204         }\r
 205         for(i = '0'; i <= '9'; i++)\r
 206         {\r
 207                 ASCIIToChrCode[i] = CHR_NUMBER;\r
 208                 ASCIIToHexDigit[i] = i-'0';\r
 209         }\r
 210         for(i = 'A'; i <= 'F'; i++)\r
 211         {\r
 212                 ASCIIToHexDigit[i] = 10+(i-'A');\r
 213         }\r
 214         for(i = 'a'; i <= 'f'; i++)\r
 215         {\r
 216                 ASCIIToHexDigit[i] = 10+(i-'a');\r
 217         }\r
 218         for(i = 'A'; i <= 'Z'; i++)\r
 219         {\r
 220                 ASCIIToChrCode[i] = CHR_LETTER;\r
 221         }\r
 222         for(i = 'a'; i <= 'z'; i++)\r
 223         {\r
 224                 ASCIIToChrCode[i] = CHR_LETTER;\r
 225         }\r
 226         ASCIIToChrCode[ASCII_QUOTE] = CHR_QUOTE;\r
 227         ASCIIToChrCode[ASCII_UNDERSCORE] = CHR_LETTER;\r
 228         ASCIIToChrCode[EOF_CHARACTER] = CHR_EOF;\r
 229         tk_String = TokenStringBuffer;\r
 230         IncLineNumber = FALSE;\r
 231         tk_IncludedLines = 0;\r
 232         SourceOpen = FALSE;\r
 233         *MasterSourceLine = '\0'; // master line - Ty 07jan2000\r
 234         MasterSourcePos = 0;      // master position - Ty 07jan2000\r
 235         ClearMasterSourceLine = TRUE; // clear the line to start\r
 236         qsort (Keywords, NUM_KEYWORDS, sizeof(Keywords[0]), SortKeywords);\r
 237         FileNames = MS_Alloc(4096, ERR_OUT_OF_MEMORY);\r
 238         FileNamesLen = 0;\r
 239         FileNamesMax = 4096;\r
 240 }\r
 241 \r
 242 //==========================================================================\r
 243 //\r
 244 // SortKeywords\r
 245 //\r
 246 //==========================================================================\r
 247 \r
 248 static int SortKeywords(const void *a, const void *b)\r
 249 {\r
 250         return strcmp (((struct keyword_s *)a)->name,\r
 251                 ((struct keyword_s *)b)->name);\r
 252 }\r
 253 \r
 254 //==========================================================================\r
 255 //\r
 256 // TK_OpenSource\r
 257 //\r
 258 //==========================================================================\r
 259 \r
 260 void TK_OpenSource(char *fileName)\r
 261 {\r
 262         int size;\r
 263 \r
 264         TK_CloseSource();\r
 265         size = MS_LoadFile(fileName, &FileStart);\r
 266         tk_SourceName = AddFileName(fileName);\r
 267         MakeIncludePath(fileName);\r
 268         SourceOpen = TRUE;\r
 269         FileEnd = FileStart+size;\r
 270         FilePtr = FileStart;\r
 271         tk_Line = 1;\r
 272         tk_Token = TK_NONE;\r
 273         AlreadyGot = FALSE;\r
 274         NestDepth = 0;\r
 275         NextChr();\r
 276 }\r
 277 \r
 278 //==========================================================================\r
 279 //\r
 280 // AddFileName\r
 281 //\r
 282 //==========================================================================\r
 283 \r
 284 static char *AddFileName(const char *name)\r
 285 {\r
 286         size_t len = strlen(name) + 1;\r
 287         char *namespot;\r
 288 \r
 289         if (FileNamesLen + len > FileNamesMax)\r
 290         {\r
 291                 FileNames = MS_Alloc(FileNamesMax, ERR_OUT_OF_MEMORY);\r
 292                 FileNamesLen = 0;\r
 293         }\r
 294         namespot = FileNames + FileNamesLen;\r
 295         memcpy(namespot, name, len);\r
 296         FileNamesLen += len;\r
 297         return namespot;\r
 298 }\r
 299 \r
 300 //==========================================================================\r
 301 //\r
 302 // MakeIncludePath\r
 303 //\r
 304 //==========================================================================\r
 305 \r
 306 static void MakeIncludePath(char *sourceName)\r
 307 {\r
 308         strcpy(IncludePath, sourceName);\r
 309         if(MS_StripFilename(IncludePath) == NO)\r
 310         {\r
 311                 IncludePath[0] = 0;\r
 312         }\r
 313         else\r
 314         { // Add a directory delimiter to the include path\r
 315                 strcat(IncludePath, DIRECTORY_DELIMITER);\r
 316         }\r
 317 }\r
 318 \r
 319 //==========================================================================\r
 320 //\r
 321 // TK_Include\r
 322 //\r
 323 //==========================================================================\r
 324 \r
 325 void TK_Include(char *fileName)\r
 326 {\r
 327         char sourceName[MAX_FILE_NAME_LENGTH];\r
 328         int size;\r
 329         nestInfo_t *info;\r
 330 \r
 331         MS_Message(MSG_DEBUG, "*Including %s\n", fileName);\r
 332         if(NestDepth == MAX_NESTED_SOURCES)\r
 333         {\r
 334                 ERR_Exit(ERR_INCL_NESTING_TOO_DEEP, YES, fileName);\r
 335         }\r
 336         info = &OpenFiles[NestDepth++];\r
 337         info->name = tk_SourceName;\r
 338         info->start = FileStart;\r
 339         info->end = FileEnd;\r
 340         info->position = FilePtr;\r
 341         info->line = tk_Line;\r
 342         info->incLineNumber = IncLineNumber;\r
 343         info->lastChar = Chr;\r
 344         info->imported = NO;\r
 345         strcpy(sourceName, IncludePath);\r
 346         strcat(sourceName, fileName);\r
 347         tk_SourceName = AddFileName(sourceName);\r
 348         size = MS_LoadFile(tk_SourceName, &FileStart);\r
 349         FileEnd = FileStart+size;\r
 350         FilePtr = FileStart;\r
 351         tk_Line = 1;\r
 352         IncLineNumber = FALSE;\r
 353         tk_Token = TK_NONE;\r
 354         AlreadyGot = FALSE;\r
 355         BumpMasterSourceLine('x',TRUE); // dummy x\r
 356         NextChr();\r
 357 }\r
 358 \r
 359 //==========================================================================\r
 360 //\r
 361 // TK_Import\r
 362 //\r
 363 //==========================================================================\r
 364 \r
 365 void TK_Import(char *fileName, enum ImportModes prevMode)\r
 366 {\r
 367         TK_Include (fileName);\r
 368         OpenFiles[NestDepth - 1].imported = YES;\r
 369         OpenFiles[NestDepth - 1].prevMode = prevMode;\r
 370         ImportMode = IMPORT_Importing;\r
 371 }\r
 372 \r
 373 //==========================================================================\r
 374 //\r
 375 // PopNestedSource\r
 376 //\r
 377 //==========================================================================\r
 378 \r
 379 static int PopNestedSource(enum ImportModes *prevMode)\r
 380 {\r
 381         nestInfo_t *info;\r
 382 \r
 383         MS_Message(MSG_DEBUG, "*Leaving %s\n", tk_SourceName);\r
 384         free(FileStart);\r
 385         tk_IncludedLines += tk_Line;\r
 386         info = &OpenFiles[--NestDepth];\r
 387         tk_SourceName = info->name;\r
 388         FileStart = info->start;\r
 389         FileEnd = info->end;\r
 390         FilePtr = info->position;\r
 391         tk_Line = info->line;\r
 392         IncLineNumber = info->incLineNumber;\r
 393         Chr = info->lastChar;\r
 394         tk_Token = TK_NONE;\r
 395         AlreadyGot = FALSE;\r
 396         *prevMode = info->prevMode;\r
 397         return info->imported ? 2 : 0;\r
 398 }\r
 399 \r
 400 //==========================================================================\r
 401 //\r
 402 // TK_CloseSource\r
 403 //\r
 404 //==========================================================================\r
 405 \r
 406 void TK_CloseSource(void)\r
 407 {\r
 408         int i;\r
 409 \r
 410         if(SourceOpen)\r
 411         {\r
 412                 free(FileStart);\r
 413                 for(i = 0; i < NestDepth; i++)\r
 414                 {\r
 415                         free(OpenFiles[i].start);\r
 416                 }\r
 417                 SourceOpen = FALSE;\r
 418         }\r
 419 }\r
 420 \r
 421 //==========================================================================\r
 422 //\r
 423 // TK_NextToken\r
 424 //\r
 425 //==========================================================================\r
 426 \r
 427 tokenType_t TK_NextToken(void)\r
 428 {\r
 429         enum ImportModes prevMode;\r
 430         boolean validToken;\r
 431 \r
 432         if(AlreadyGot == TRUE)\r
 433         {\r
 434                 int t = MasterSourcePos;\r
 435                 MasterSourcePos = PrevMasterSourcePos;\r
 436                 PrevMasterSourcePos = t;\r
 437                 AlreadyGot = FALSE;\r
 438                 return tk_Token;\r
 439         }\r
 440         validToken = NO;\r
 441         PrevMasterSourcePos = MasterSourcePos;\r
 442         do\r
 443         {\r
 444                 while(Chr == ASCII_SPACE)\r
 445                 {\r
 446                         NextChr();\r
 447                 }\r
 448                 switch(ASCIIToChrCode[(byte)Chr])\r
 449                 {\r
 450                 case CHR_EOF:\r
 451                         tk_Token = TK_EOF;\r
 452                         break;\r
 453                 case CHR_LETTER:\r
 454                         ProcessLetterToken();\r
 455                         break;\r
 456                 case CHR_NUMBER:\r
 457                         ProcessNumberToken();\r
 458                         break;\r
 459                 case CHR_QUOTE:\r
 460                         ProcessQuoteToken();\r
 461                         break;\r
 462                 default:\r
 463                         ProcessSpecialToken();\r
 464                         break;\r
 465                 }\r
 466                 if(tk_Token == TK_STARTCOMMENT)\r
 467                 {\r
 468                         SkipComment();\r
 469                 }\r
 470                 else if(tk_Token == TK_CPPCOMMENT)\r
 471                 {\r
 472                         SkipCPPComment();\r
 473                 }\r
 474                 else if((tk_Token == TK_EOF) && (NestDepth > 0))\r
 475                 {\r
 476                         if (PopNestedSource(&prevMode))\r
 477                         {\r
 478                                 ImportMode = prevMode;\r
 479                                 if(!ExporterFlagged)\r
 480                                 {\r
 481                                         ERR_Exit(ERR_EXPORTER_NOT_FLAGGED, NO);\r
 482                                 }\r
 483                                 SY_ClearShared();\r
 484                         }\r
 485                 }\r
 486                 else\r
 487                 {\r
 488                         validToken = YES;\r
 489                 }\r
 490         } while(validToken == NO);\r
 491         return tk_Token;\r
 492 }\r
 493 \r
 494 //==========================================================================\r
 495 //\r
 496 // TK_NextCharacter\r
 497 //\r
 498 //==========================================================================\r
 499 \r
 500 int TK_NextCharacter(void)\r
 501 {\r
 502         int c;\r
 503 \r
 504         while(Chr == ASCII_SPACE)\r
 505         {\r
 506                 NextChr();\r
 507         }\r
 508         c = (int)Chr;\r
 509         if(c == EOF_CHARACTER)\r
 510         {\r
 511                 c = -1;\r
 512         }\r
 513         NextChr();\r
 514         return c;\r
 515 }\r
 516 \r
 517 //==========================================================================\r
 518 //\r
 519 // TK_SkipPast\r
 520 //\r
 521 //==========================================================================\r
 522 \r
 523 void TK_SkipPast(tokenType_t token)\r
 524 {\r
 525         while (tk_Token != token)\r
 526         {\r
 527                 TK_NextToken();\r
 528         }\r
 529         TK_NextToken();\r
 530 }\r
 531 \r
 532 //==========================================================================\r
 533 //\r
 534 // TK_SkipTo\r
 535 //\r
 536 //==========================================================================\r
 537 \r
 538 void TK_SkipTo(tokenType_t token)\r
 539 {\r
 540         while (tk_Token != token)\r
 541         {\r
 542                 TK_NextToken();\r
 543         }\r
 544 }\r
 545 \r
 546 //==========================================================================\r
 547 //\r
 548 // TK_NextTokenMustBe\r
 549 //\r
 550 //==========================================================================\r
 551 \r
 552 boolean TK_NextTokenMustBe(tokenType_t token, error_t error)\r
 553 {\r
 554         if(TK_NextToken() != token)\r
 555         {\r
 556                 ERR_Error(error, YES);\r
 557                 /*\r
 558                 if(skipToken == TK_EOF)\r
 559                 {\r
 560                         ERR_Finish();\r
 561                 }\r
 562                 else if(skipToken != TK_NONE)\r
 563                 {\r
 564                         TK_SkipPast(skipToken);\r
 565                 }\r
 566                 return NO;\r
 567                 */\r
 568                 ERR_Finish();\r
 569         }\r
 570         return YES;\r
 571 }\r
 572 \r
 573 //==========================================================================\r
 574 //\r
 575 // TK_TokenMustBe\r
 576 //\r
 577 //==========================================================================\r
 578 \r
 579 boolean TK_TokenMustBe(tokenType_t token, error_t error)\r
 580 {\r
 581         if (token == TK_SEMICOLON && forSemicolonHack)\r
 582         {\r
 583                 token = TK_RPAREN;\r
 584         }\r
 585         if(tk_Token != token)\r
 586         {\r
 587                 ERR_Error(error, YES);\r
 588                 /*\r
 589                 if(skipToken == TK_EOF)\r
 590                 {\r
 591                         ERR_Finish();\r
 592                 }\r
 593                 else if(skipToken != TK_NONE)\r
 594                 {\r
 595                         while(tk_Token != skipToken)\r
 596                         {\r
 597                                 TK_NextToken();\r
 598                         }\r
 599                 }\r
 600                 return NO;\r
 601                 */\r
 602                 ERR_Finish();\r
 603         }\r
 604         return YES;\r
 605 }\r
 606 \r
 607 //==========================================================================\r
 608 //\r
 609 // TK_Member\r
 610 //\r
 611 //==========================================================================\r
 612 \r
 613 boolean TK_Member(tokenType_t *list)\r
 614 {\r
 615         int i;\r
 616 \r
 617         for(i = 0; list[i] != TK_NONE; i++)\r
 618         {\r
 619                 if(tk_Token == list[i])\r
 620                 {\r
 621                         return YES;\r
 622                 }\r
 623         }\r
 624         return NO;\r
 625 }\r
 626 \r
 627 //==========================================================================\r
 628 //\r
 629 // TK_Undo\r
 630 //\r
 631 //==========================================================================\r
 632 \r
 633 void TK_Undo(void)\r
 634 {\r
 635         if(tk_Token != TK_NONE)\r
 636         {\r
 637                 if (AlreadyGot == FALSE)\r
 638                 {\r
 639                         int t = MasterSourcePos;\r
 640                         MasterSourcePos = PrevMasterSourcePos;\r
 641                         PrevMasterSourcePos = t;\r
 642                         AlreadyGot = TRUE;\r
 643                 }\r
 644         }\r
 645 }\r
 646 \r
 647 //==========================================================================\r
 648 //\r
 649 // ProcessLetterToken\r
 650 //\r
 651 //==========================================================================\r
 652 \r
 653 static void ProcessLetterToken(void)\r
 654 {\r
 655         int i;\r
 656         char *text;\r
 657 \r
 658         i = 0;\r
 659         text = TokenStringBuffer;\r
 660         while (ASCIIToChrCode[(byte)Chr] == CHR_LETTER\r
 661                 || ASCIIToChrCode[(byte)Chr] == CHR_NUMBER)\r
 662         {\r
 663                 if(++i == MAX_IDENTIFIER_LENGTH)\r
 664                 {\r
 665                         ERR_Error(ERR_IDENTIFIER_TOO_LONG, YES);\r
 666                 }\r
 667                 if(i < MAX_IDENTIFIER_LENGTH)\r
 668                 {\r
 669                         *text++ = Chr;\r
 670                 }\r
 671                 NextChr();\r
 672         }\r
 673         *text = 0;\r
 674         MS_StrLwr(TokenStringBuffer);\r
 675         if(CheckForKeyword() == FALSE\r
 676                 && CheckForLineSpecial() == FALSE\r
 677                 && CheckForConstant() == FALSE)\r
 678         {\r
 679                 tk_Token = TK_IDENTIFIER;\r
 680         }\r
 681 }\r
 682 \r
 683 //==========================================================================\r
 684 //\r
 685 // CheckForKeyword\r
 686 //\r
 687 //==========================================================================\r
 688 \r
 689 static boolean CheckForKeyword(void)\r
 690 {\r
 691         int min, max, probe, lexx;\r
 692 \r
 693         // [RH] Use a binary search\r
 694         min = 0;\r
 695         max = NUM_KEYWORDS-1;\r
 696         probe = NUM_KEYWORDS/2;\r
 697 \r
 698         while (max - min >= 0)\r
 699         {\r
 700                 lexx = strcmp(tk_String, Keywords[probe].name);\r
 701                 if(lexx == 0)\r
 702                 {\r
 703                         tk_Token = Keywords[probe].token;\r
 704                         return TRUE;\r
 705                 }\r
 706                 else if(lexx < 0)\r
 707                 {\r
 708                         max = probe-1;\r
 709                 }\r
 710                 else\r
 711                 {\r
 712                         min = probe+1;\r
 713                 }\r
 714                 probe = (max-min)/2+min;\r
 715         }\r
 716         return FALSE;\r
 717 }\r
 718 \r
 719 //==========================================================================\r
 720 //\r
 721 // CheckForLineSpecial\r
 722 //\r
 723 //==========================================================================\r
 724 \r
 725 static boolean CheckForLineSpecial(void)\r
 726 {\r
 727         symbolNode_t *sym;\r
 728 \r
 729         sym = SY_FindGlobal(tk_String);\r
 730         if(sym == NULL)\r
 731         {\r
 732                 return FALSE;\r
 733         }\r
 734         if(sym->type != SY_SPECIAL)\r
 735         {\r
 736                 return FALSE;\r
 737         }\r
 738         tk_Token = TK_LINESPECIAL;\r
 739         tk_SpecialValue = sym->info.special.value;\r
 740         tk_SpecialArgCount = sym->info.special.argCount;\r
 741         return TRUE;\r
 742 }\r
 743 \r
 744 //==========================================================================\r
 745 //\r
 746 // CheckForConstant\r
 747 //\r
 748 //==========================================================================\r
 749 \r
 750 static boolean CheckForConstant(void)\r
 751 {\r
 752         symbolNode_t *sym;\r
 753 \r
 754         sym = SY_FindGlobal(tk_String);\r
 755         if(sym == NULL)\r
 756         {\r
 757                 return FALSE;\r
 758         }\r
 759         if(sym->type != SY_CONSTANT)\r
 760         {\r
 761                 return FALSE;\r
 762         }\r
 763         tk_Token = TK_NUMBER;\r
 764         tk_Number = sym->info.constant.value;\r
 765         return TRUE;\r
 766 }\r
 767 \r
 768 //==========================================================================\r
 769 //\r
 770 // ProcessNumberToken\r
 771 //\r
 772 //==========================================================================\r
 773 \r
 774 static void ProcessNumberToken(void)\r
 775 {\r
 776         char c;\r
 777 \r
 778         c = Chr;\r
 779         NextChr();\r
 780         if(c == '0' && (Chr == 'x' || Chr == 'X'))\r
 781         { // Hexadecimal constant\r
 782                 NextChr();\r
 783                 EvalHexConstant();\r
 784                 return;\r
 785         }\r
 786         tk_Number = c-'0';\r
 787         while(ASCIIToChrCode[(byte)Chr] == CHR_NUMBER)\r
 788         {\r
 789                 tk_Number = 10*tk_Number+(Chr-'0');\r
 790                 NextChr();\r
 791         }\r
 792         if(Chr == '.')\r
 793         { // Fixed point\r
 794                 NextChr(); // Skip period\r
 795                 EvalFixedConstant(tk_Number);\r
 796                 return;\r
 797         }\r
 798         if(Chr == ASCII_UNDERSCORE)\r
 799         {\r
 800                 NextChr(); // Skip underscore\r
 801                 EvalRadixConstant();\r
 802                 return;\r
 803         }\r
 804         tk_Token = TK_NUMBER;\r
 805 }\r
 806 \r
 807 //==========================================================================\r
 808 //\r
 809 // EvalFixedConstant\r
 810 //\r
 811 //==========================================================================\r
 812 \r
 813 static void EvalFixedConstant(int whole)\r
 814 {\r
 815         int frac;\r
 816         int divisor;\r
 817 \r
 818         frac = 0;\r
 819         divisor = 1;\r
 820         while(ASCIIToChrCode[(byte)Chr] == CHR_NUMBER)\r
 821         {\r
 822                 frac = 10*frac+(Chr-'0');\r
 823                 divisor *= 10;\r
 824                 NextChr();\r
 825         }\r
 826         tk_Number = (whole<<16)+((frac<<16)/divisor);\r
 827         tk_Token = TK_NUMBER;\r
 828 }\r
 829 \r
 830 //==========================================================================\r
 831 //\r
 832 // EvalHexConstant\r
 833 //\r
 834 //==========================================================================\r
 835 \r
 836 static void EvalHexConstant(void)\r
 837 {\r
 838         tk_Number = 0;\r
 839         while(ASCIIToHexDigit[(byte)Chr] != NON_HEX_DIGIT)\r
 840         {\r
 841                 tk_Number = (tk_Number<<4)+ASCIIToHexDigit[(byte)Chr];\r
 842                 NextChr();\r
 843         }\r
 844         tk_Token = TK_NUMBER;\r
 845 }\r
 846 \r
 847 //==========================================================================\r
 848 //\r
 849 // EvalRadixConstant\r
 850 //\r
 851 //==========================================================================\r
 852 \r
 853 static void EvalRadixConstant(void)\r
 854 {\r
 855         int radix;\r
 856         int digitVal;\r
 857 \r
 858         radix = tk_Number;\r
 859         if(radix < 2 || radix > 36)\r
 860         {\r
 861                 ERR_Error(ERR_BAD_RADIX_CONSTANT, YES, NULL);\r
 862                 radix = 36;\r
 863         }\r
 864         tk_Number = 0;\r
 865         while((digitVal = DigitValue(Chr, radix)) != -1)\r
 866         {\r
 867                 tk_Number = radix*tk_Number+digitVal;\r
 868                 NextChr();\r
 869         }\r
 870         tk_Token = TK_NUMBER;\r
 871 }\r
 872 \r
 873 //==========================================================================\r
 874 //\r
 875 // DigitValue\r
 876 //\r
 877 // Returns -1 if the digit is not allowed in the specified radix.\r
 878 //\r
 879 //==========================================================================\r
 880 \r
 881 static int DigitValue(char digit, int radix)\r
 882 {\r
 883         digit = toupper(digit);\r
 884         if(digit < '0' || (digit > '9' && digit < 'A') || digit > 'Z')\r
 885         {\r
 886                 return -1;\r
 887         }\r
 888         if(digit > '9')\r
 889         {\r
 890                 digit = 10+digit-'A';\r
 891         }\r
 892         else\r
 893         {\r
 894                 digit -= '0';\r
 895         }\r
 896         if(digit >= radix)\r
 897         {\r
 898                 return -1;\r
 899         }\r
 900         return digit;\r
 901 }\r
 902 \r
 903 //==========================================================================\r
 904 //\r
 905 // ProcessQuoteToken\r
 906 //\r
 907 //==========================================================================\r
 908 \r
 909 static void ProcessQuoteToken(void)\r
 910 {\r
 911         int i;\r
 912         char *text;\r
 913         boolean escaped;\r
 914 \r
 915         i = 0;\r
 916         escaped = FALSE;\r
 917         text = TokenStringBuffer;\r
 918         NextChr();\r
 919         while(Chr != EOF_CHARACTER)\r
 920         {\r
 921                 if(Chr == ASCII_QUOTE && escaped == 0) // [JB]\r
 922                 {\r
 923                         break;\r
 924                 }\r
 925                 if(++i == MAX_QUOTED_LENGTH)\r
 926                 {\r
 927                         ERR_Error(ERR_STRING_TOO_LONG, YES, NULL);\r
 928                 }\r
 929                 if(i < MAX_QUOTED_LENGTH)\r
 930                 {\r
 931                         *text++ = Chr;\r
 932                 }\r
 933                 // escape the character after a backslash [JB]\r
 934                 if(Chr == ASCII_BACKSLASH)\r
 935                         escaped ^= (Chr == ASCII_BACKSLASH);\r
 936                 else\r
 937                         escaped = FALSE;\r
 938                 NextChr();\r
 939         }\r
 940         *text = 0;\r
 941         if(Chr == ASCII_QUOTE)\r
 942         {\r
 943                 NextChr();\r
 944         }\r
 945         tk_Token = TK_STRING;\r
 946 }\r
 947 \r
 948 //==========================================================================\r
 949 //\r
 950 // ProcessSpecialToken\r
 951 //\r
 952 //==========================================================================\r
 953 \r
 954 static void ProcessSpecialToken(void)\r
 955 {\r
 956         char c;\r
 957 \r
 958         c = Chr;\r
 959         NextChr();\r
 960         switch(c)\r
 961         {\r
 962                 case '+':\r
 963                         switch(Chr)\r
 964                         {\r
 965                                 case '=':\r
 966                                         tk_Token = TK_ADDASSIGN;\r
 967                                         NextChr();\r
 968                                         break;\r
 969                                 case '+':\r
 970                                         tk_Token = TK_INC;\r
 971                                         NextChr();\r
 972                                         break;\r
 973                                 default:\r
 974                                         tk_Token = TK_PLUS;\r
 975                                         break;\r
 976                         }\r
 977                         break;\r
 978                 case '-':\r
 979                         switch(Chr)\r
 980                         {\r
 981                                 case '=':\r
 982                                         tk_Token = TK_SUBASSIGN;\r
 983                                         NextChr();\r
 984                                         break;\r
 985                                 case '-':\r
 986                                         tk_Token = TK_DEC;\r
 987                                         NextChr();\r
 988                                         break;\r
 989                                 default:\r
 990                                         tk_Token = TK_MINUS;\r
 991                                         break;\r
 992                         }\r
 993                         break;\r
 994                 case '*':\r
 995                         switch(Chr)\r
 996                         {\r
 997                                 case '=':\r
 998                                         tk_Token = TK_MULASSIGN;\r
 999                                         NextChr();\r
1000                                         break;\r
1001                                 case '/':\r
1002                                         tk_Token = TK_ENDCOMMENT;\r
1003                                         NextChr();\r
1004                                         break;\r
1005                                 default:\r
1006                                         tk_Token = TK_ASTERISK;\r
1007                                         break;\r
1008                         }\r
1009                         break;\r
1010                 case '/':\r
1011                         switch(Chr)\r
1012                         {\r
1013                                 case '=':\r
1014                                         tk_Token = TK_DIVASSIGN;\r
1015                                         NextChr();\r
1016                                         break;\r
1017                                 case '/':\r
1018                                         tk_Token = TK_CPPCOMMENT;\r
1019                                         break;\r
1020                                 case '*':\r
1021                                         tk_Token = TK_STARTCOMMENT;\r
1022                                         NextChr();\r
1023                                         break;\r
1024                                 default:\r
1025                                         tk_Token = TK_SLASH;\r
1026                                         break;\r
1027                         }\r
1028                         break;\r
1029                 case '%':\r
1030                         if(Chr == '=')\r
1031                         {\r
1032                                 tk_Token = TK_MODASSIGN;\r
1033                                 NextChr();\r
1034                         }\r
1035                         else\r
1036                         {\r
1037                                 tk_Token = TK_PERCENT;\r
1038                         }\r
1039                         break;\r
1040                 case '=':\r
1041                         if(Chr == '=')\r
1042                         {\r
1043                                 tk_Token = TK_EQ;\r
1044                                 NextChr();\r
1045                         }\r
1046                         else\r
1047                         {\r
1048                                 tk_Token = TK_ASSIGN;\r
1049                         }\r
1050                         break;\r
1051                 case '<':\r
1052                         if(Chr == '=')\r
1053                         {\r
1054                                 tk_Token = TK_LE;\r
1055                                 NextChr();\r
1056                         }\r
1057                         else if(Chr == '<')\r
1058                         {\r
1059                                 tk_Token = TK_LSHIFT;\r
1060                                 NextChr();\r
1061                         }\r
1062                         else\r
1063                         {\r
1064                                 tk_Token = TK_LT;\r
1065                         }\r
1066                         break;\r
1067                 case '>':\r
1068                         if(Chr == '=')\r
1069                         {\r
1070                                 tk_Token = TK_GE;\r
1071                                 NextChr();\r
1072                         }\r
1073                         else if(Chr == '>')\r
1074                         {\r
1075                                 tk_Token = TK_RSHIFT;\r
1076                                 NextChr();\r
1077                         }\r
1078                         else\r
1079                         {\r
1080                                 tk_Token = TK_GT;\r
1081                         }\r
1082                         break;\r
1083                 case '!':\r
1084                         if(Chr == '=')\r
1085                         {\r
1086                                 tk_Token = TK_NE;\r
1087                                 NextChr();\r
1088                         }\r
1089                         else\r
1090                         {\r
1091                                 tk_Token = TK_NOT;\r
1092                         }\r
1093                         break;\r
1094                 case '&':\r
1095                         if(Chr == '&')\r
1096                         {\r
1097                                 tk_Token = TK_ANDLOGICAL;\r
1098                                 NextChr();\r
1099                         }\r
1100                         else\r
1101                         {\r
1102                                 tk_Token = TK_ANDBITWISE;\r
1103                         }\r
1104                         break;\r
1105                 case '|':\r
1106                         if(Chr == '|')\r
1107                         {\r
1108                                 tk_Token = TK_ORLOGICAL;\r
1109                                 NextChr();\r
1110                         }\r
1111                         else\r
1112                         {\r
1113                                 tk_Token = TK_ORBITWISE;\r
1114                         }\r
1115                         break;\r
1116                 case '(':\r
1117                         tk_Token = TK_LPAREN;\r
1118                         break;\r
1119                 case ')':\r
1120                         tk_Token = TK_RPAREN;\r
1121                         break;\r
1122                 case '{':\r
1123                         tk_Token = TK_LBRACE;\r
1124                         break;\r
1125                 case '}':\r
1126                         tk_Token = TK_RBRACE;\r
1127                         break;\r
1128                 case '[':\r
1129                         tk_Token = TK_LBRACKET;\r
1130                         break;\r
1131                 case ']':\r
1132                         tk_Token = TK_RBRACKET;\r
1133                         break;\r
1134                 case ':':\r
1135                         tk_Token = TK_COLON;\r
1136                         break;\r
1137                 case ';':\r
1138                         tk_Token = TK_SEMICOLON;\r
1139                         break;\r
1140                 case ',':\r
1141                         tk_Token = TK_COMMA;\r
1142                         break;\r
1143                 case '.':\r
1144                         tk_Token = TK_PERIOD;\r
1145                         break;\r
1146                 case '#':\r
1147                         tk_Token = TK_NUMBERSIGN;\r
1148                         break;\r
1149                 case '^':\r
1150                         tk_Token = TK_EORBITWISE;\r
1151                         break;\r
1152                 case '~':\r
1153                         tk_Token = TK_TILDE;\r
1154                         break;\r
1155                 case '\'':\r
1156                         if(Chr == '\\')\r
1157                         {\r
1158                                 NextChr();\r
1159                                 switch(Chr)\r
1160                                 {\r
1161                                 case '0': case '1': case '2': case '3':\r
1162                                 case '4': case '5': case '6': case '7':\r
1163                                         tk_Number = OctalChar();\r
1164                                         break;\r
1165                                 case 'x': case 'X':\r
1166                                         NextChr();\r
1167                                         EvalHexConstant();\r
1168                                         if(Chr != '\'')\r
1169                                         {\r
1170                                                 ERR_Exit(ERR_BAD_CHARACTER_CONSTANT, YES, NULL);\r
1171                                         }\r
1172                                         NextChr();\r
1173                                         break;\r
1174                                 case 'a':\r
1175                                         tk_Number = '\a';\r
1176                                         break;\r
1177                                 case 'b':\r
1178                                         tk_Number = '\b';\r
1179                                         break;\r
1180                                 case 't':\r
1181                                         tk_Number = '\t';\r
1182                                         break;\r
1183                                 case 'v':\r
1184                                         tk_Number = '\v';\r
1185                                         break;\r
1186                                 case 'n':\r
1187                                         tk_Number = '\n';\r
1188                                         break;\r
1189                                 case 'f':\r
1190                                         tk_Number = '\f';\r
1191                                         break;\r
1192                                 case 'r':\r
1193                                         tk_Number = '\r';\r
1194                                         break;\r
1195                                 case '\'':\r
1196                                         tk_Number = Chr;\r
1197                                         break;\r
1198                                 default:\r
1199                                         ERR_Exit(ERR_BAD_CHARACTER_CONSTANT, YES, NULL);\r
1200                                 }\r
1201                                 tk_Token = TK_NUMBER;\r
1202                         }\r
1203                         else if(Chr == '\'')\r
1204                         {\r
1205                                 ERR_Exit(ERR_BAD_CHARACTER_CONSTANT, YES, NULL);\r
1206                         }\r
1207                         else\r
1208                         {\r
1209                                 tk_Number = Chr;\r
1210                                 tk_Token = TK_NUMBER;\r
1211                         }\r
1212                         NextChr();\r
1213                         if(Chr != '\'')\r
1214                         {\r
1215                                 ERR_Exit(ERR_BAD_CHARACTER_CONSTANT, YES, NULL);\r
1216                         }\r
1217                         NextChr();\r
1218                         break;\r
1219                 default:\r
1220                         ERR_Exit(ERR_BAD_CHARACTER, YES, NULL);\r
1221                         break;\r
1222         }\r
1223 }\r
1224 \r
1225 //==========================================================================\r
1226 //\r
1227 // NextChr\r
1228 //\r
1229 //==========================================================================\r
1230 \r
1231 static void NextChr(void)\r
1232 {\r
1233         if(FilePtr >= FileEnd)\r
1234         {\r
1235                 Chr = EOF_CHARACTER;\r
1236                 return;\r
1237         }\r
1238         if(IncLineNumber == TRUE)\r
1239         {\r
1240                 tk_Line++;\r
1241                 IncLineNumber = FALSE;\r
1242                 BumpMasterSourceLine('x',TRUE); // dummy x\r
1243         }\r
1244         Chr = *FilePtr++;\r
1245         if(Chr < ASCII_SPACE && Chr >= 0)       // Allow high ASCII characters\r
1246         {\r
1247                 if(Chr == '\n')\r
1248                 {\r
1249                         IncLineNumber = TRUE;\r
1250                 }\r
1251                 Chr = ASCII_SPACE;\r
1252         }\r
1253         BumpMasterSourceLine(Chr,FALSE);\r
1254 }\r
1255 \r
1256 //==========================================================================\r
1257 //\r
1258 // PeekChr // [JB]\r
1259 //\r
1260 //==========================================================================\r
1261 \r
1262 static int PeekChr(void)\r
1263 {\r
1264         char ch;\r
1265         if(FilePtr >= FileEnd)\r
1266         {\r
1267                 return EOF_CHARACTER;\r
1268         }\r
1269         ch = *FilePtr-1;\r
1270         if(ch < ASCII_SPACE && ch >= 0) // Allow high ASCII characters\r
1271         {\r
1272                 ch = ASCII_SPACE;\r
1273         }\r
1274         return ch;\r
1275 }\r
1276 \r
1277 //==========================================================================\r
1278 //\r
1279 // OctalChar // [JB]\r
1280 //\r
1281 //==========================================================================\r
1282 \r
1283 static int OctalChar() \r
1284 {\r
1285         int digits = 1;\r
1286         int code = Chr - '0';\r
1287         while(digits < 4 && PeekChr() >= '0' && PeekChr() <= '7')\r
1288         {\r
1289                 NextChr();\r
1290                 code = (code << 3) + Chr - '0';\r
1291         }\r
1292         return code;\r
1293 }\r
1294 \r
1295 //==========================================================================\r
1296 //\r
1297 // SkipComment\r
1298 //\r
1299 //==========================================================================\r
1300 \r
1301 void SkipComment(void)\r
1302 {\r
1303         boolean first;\r
1304 \r
1305         first = FALSE;\r
1306         while(Chr != EOF_CHARACTER)\r
1307         {\r
1308                 if(first == TRUE && Chr == '/')\r
1309                 {\r
1310                         break;\r
1311                 }\r
1312                 first = (Chr == '*');\r
1313                 NextChr();\r
1314         }\r
1315         NextChr();\r
1316 }\r
1317 \r
1318 //==========================================================================\r
1319 //\r
1320 // SkipCPPComment\r
1321 //\r
1322 //==========================================================================\r
1323 \r
1324 void SkipCPPComment(void)\r
1325 {\r
1326         while(FilePtr < FileEnd)\r
1327         {\r
1328                 if(*FilePtr++ == '\n')\r
1329                 {\r
1330                         tk_Line++;\r
1331                         BumpMasterSourceLine('x',TRUE); // dummy x\r
1332                         break;\r
1333                 }\r
1334         }\r
1335         NextChr();\r
1336 }\r
1337 \r
1338 //==========================================================================\r
1339 //\r
1340 // BumpMasterSourceLine\r
1341 //\r
1342 //==========================================================================\r
1343 \r
1344 void BumpMasterSourceLine(char Chr, boolean clear) // master line - Ty 07jan2000\r
1345 {\r
1346         if (ClearMasterSourceLine)  // set to clear last time, clear now for first character\r
1347         {\r
1348                 *MasterSourceLine = '\0';\r
1349                 MasterSourcePos = 0;\r
1350                 ClearMasterSourceLine = FALSE;\r
1351         }\r
1352         if (clear)\r
1353         {\r
1354                 ClearMasterSourceLine = TRUE;\r
1355         }\r
1356         else\r
1357         {\r
1358                 if (MasterSourcePos < MAX_STATEMENT_LENGTH)\r
1359                         MasterSourceLine[MasterSourcePos++] = Chr;\r
1360         }\r
1361 }\r
1362 \r
1363 //==========================================================================\r
1364 //\r
1365 // TK_SkipLine\r
1366 //\r
1367 //==========================================================================\r
1368 \r
1369 void TK_SkipLine(void)\r
1370 {\r
1371         char *sourcenow = tk_SourceName;\r
1372         int linenow = tk_Line;\r
1373         do TK_NextToken(); while (tk_Line == linenow && tk_SourceName == sourcenow);\r
1374 }\r