token.c

   1 \r
   2 //**************************************************************************\r
   3 //**\r
   4 //** token.c\r
   5 //**\r
   6 //**************************************************************************\r
   7 \r
   8 // HEADER FILES ------------------------------------------------------------\r
   9 \r
  10 #ifdef __NeXT__\r
  11 #include <libc.h>\r
  12 #else\r
  13 #ifndef unix\r
  14 #include <io.h>\r
  15 #endif\r
  16 #include <fcntl.h>\r
  17 #include <stdlib.h>\r
  18 #endif\r
  19 #include <stdio.h>\r
  20 #include <string.h>\r
  21 #include <ctype.h>\r
  22 #include "common.h"\r
  23 #include "token.h"\r
  24 #include "error.h"\r
  25 #include "misc.h"\r
  26 #include "symbol.h"\r
  27 #include "parse.h"\r
  28 \r
  29 // MACROS ------------------------------------------------------------------\r
  30 \r
  31 #define NON_HEX_DIGIT 255\r
  32 #define MAX_NESTED_SOURCES 16\r
  33 \r
  34 // TYPES -------------------------------------------------------------------\r
  35 \r
  36 typedef enum\r
  37 {\r
  38         CHR_EOF,\r
  39         CHR_LETTER,\r
  40         CHR_NUMBER,\r
  41         CHR_QUOTE,\r
  42         CHR_SPECIAL\r
  43 } chr_t;\r
  44 \r
  45 typedef struct\r
  46 {\r
  47         char *name;\r
  48         char *start;\r
  49         char *end;\r
  50         char *position;\r
  51         int line;\r
  52         boolean incLineNumber;\r
  53         boolean imported;\r
  54         enum ImportModes prevMode;\r
  55         char lastChar;\r
  56 } nestInfo_t;\r
  57 \r
  58 // EXTERNAL FUNCTION PROTOTYPES --------------------------------------------\r
  59 \r
  60 // PUBLIC FUNCTION PROTOTYPES ----------------------------------------------\r
  61 \r
  62 // PRIVATE FUNCTION PROTOTYPES ---------------------------------------------\r
  63 \r
  64 static int SortKeywords(const void *a, const void *b);\r
  65 static void MakeIncludePath(char *sourceName);\r
  66 static int PopNestedSource(enum ImportModes *prevMode);\r
  67 static void ProcessLetterToken(void);\r
  68 static void ProcessNumberToken(void);\r
  69 static void EvalFixedConstant(int whole);\r
  70 static void EvalHexConstant(void);\r
  71 static void EvalRadixConstant(void);\r
  72 static int DigitValue(char digit, int radix);\r
  73 static void ProcessQuoteToken(void);\r
  74 static void ProcessSpecialToken(void);\r
  75 static boolean CheckForKeyword(void);\r
  76 static boolean CheckForLineSpecial(void);\r
  77 static boolean CheckForConstant(void);\r
  78 static void NextChr(void);\r
  79 static void SkipComment(void);\r
  80 static void SkipCPPComment(void);\r
  81 static void BumpMasterSourceLine(char Chr, boolean clear); // master line - Ty 07jan2000\r
  82 static char *AddFileName(const char *name);\r
  83 static int OctalChar();\r
  84 \r
  85 // EXTERNAL DATA DECLARATIONS ----------------------------------------------\r
  86 \r
  87 // PUBLIC DATA DEFINITIONS -------------------------------------------------\r
  88 \r
  89 tokenType_t tk_Token;\r
  90 int tk_Line;\r
  91 int tk_Number;\r
  92 char *tk_String;\r
  93 U_BYTE tk_SpecialValue;\r
  94 int tk_SpecialArgCount;\r
  95 char *tk_SourceName;\r
  96 int tk_IncludedLines;\r
  97 boolean forSemicolonHack;\r
  98 char MasterSourceLine[MAX_STATEMENT_LENGTH+1];  // master line - Ty 07jan2000\r
  99 int MasterSourcePos;      // master position - Ty 07jan2000\r
 100 int PrevMasterSourcePos;        // previous master position - RH 09feb2000\r
 101 boolean ClearMasterSourceLine;  // master clear flag - Ty 07jan2000\r
 102 \r
 103 // PRIVATE DATA DEFINITIONS ------------------------------------------------\r
 104 \r
 105 static char Chr;\r
 106 static char *FileStart;\r
 107 static char *FilePtr;\r
 108 static char *FileEnd;\r
 109 static boolean SourceOpen;\r
 110 static char ASCIIToChrCode[256];\r
 111 static byte ASCIIToHexDigit[256];\r
 112 static char TokenStringBuffer[MAX_QUOTED_LENGTH];\r
 113 static nestInfo_t OpenFiles[MAX_NESTED_SOURCES];\r
 114 static boolean AlreadyGot;\r
 115 static int NestDepth;\r
 116 static boolean IncLineNumber;\r
 117 static char IncludePath[MAX_FILE_NAME_LENGTH];\r
 118 static char *FileNames;\r
 119 static size_t FileNamesLen, FileNamesMax;\r
 120 \r
 121 static struct keyword_s\r
 122 {\r
 123         char *name;\r
 124         tokenType_t token;\r
 125 } Keywords[] =\r
 126 {\r
 127         { "break", TK_BREAK },\r
 128         { "case", TK_CASE },\r
 129         { "const", TK_CONST },\r
 130         { "continue", TK_CONTINUE },\r
 131         { "default", TK_DEFAULT },\r
 132         { "define", TK_DEFINE },\r
 133         { "do", TK_DO },\r
 134         { "else", TK_ELSE },\r
 135         { "for", TK_FOR },\r
 136         { "goto", TK_GOTO },\r
 137         { "if", TK_IF },\r
 138         { "include", TK_INCLUDE },\r
 139         { "int", TK_INT },\r
 140         { "open", TK_OPEN },\r
 141         { "print", TK_PRINT },\r
 142         { "printbold", TK_PRINTBOLD },\r
 143         { "log", TK_LOG },\r
 144         { "hudmessage", TK_HUDMESSAGE },\r
 145         { "hudmessagebold", TK_HUDMESSAGEBOLD },\r
 146         { "restart", TK_RESTART },\r
 147         { "script", TK_SCRIPT },\r
 148         { "special", TK_SPECIAL },\r
 149         { "str", TK_STR },\r
 150         { "suspend", TK_SUSPEND },\r
 151         { "switch", TK_SWITCH },\r
 152         { "terminate", TK_TERMINATE },\r
 153         { "until", TK_UNTIL },\r
 154         { "void", TK_VOID },\r
 155         { "while", TK_WHILE },\r
 156         { "world", TK_WORLD },\r
 157         { "global", TK_GLOBAL },\r
 158         // [BC] Start Skulltag tokens.\r
 159         { "respawn", TK_RESPAWN },\r
 160         { "death", TK_DEATH },\r
 161         { "enter", TK_ENTER },\r
 162         { "pickup", TK_PICKUP },\r
 163         { "bluereturn", TK_BLUERETURN },\r
 164         { "redreturn", TK_REDRETURN },\r
 165         { "whitereturn", TK_WHITERETURN },\r
 166         // [BC] End Skulltag tokens.\r
 167         { "nocompact", TK_NOCOMPACT },\r
 168         { "lightning", TK_LIGHTNING },\r
 169         { "createtranslation", TK_CREATETRANSLATION },\r
 170         { "function", TK_FUNCTION },\r
 171         { "return", TK_RETURN },\r
 172         { "wadauthor", TK_WADAUTHOR },\r
 173         { "nowadauthor", TK_NOWADAUTHOR },\r
 174         { "acs_executewait", TK_ACSEXECUTEWAIT },\r
 175         { "encryptstrings", TK_ENCRYPTSTRINGS },\r
 176         { "import", TK_IMPORT },\r
 177         { "library", TK_LIBRARY },\r
 178         { "libdefine", TK_LIBDEFINE },\r
 179         { "bool", TK_BOOL },\r
 180         { "net", TK_NET },\r
 181         { "disconnect", TK_DISCONNECT },\r
 182         { "unloading", TK_UNLOADING },\r
 183         { "static", TK_STATIC }\r
 184 };\r
 185 \r
 186 #define NUM_KEYWORDS (sizeof(Keywords)/sizeof(Keywords[0]))\r
 187 \r
 188 // CODE --------------------------------------------------------------------\r
 189 \r
 190 //==========================================================================\r
 191 //\r
 192 // TK_Init\r
 193 //\r
 194 //==========================================================================\r
 195 \r
 196 void TK_Init(void)\r
 197 {\r
 198         int i;\r
 199 \r
 200         for(i = 0; i < 256; i++)\r
 201         {\r
 202                 ASCIIToChrCode[i] = CHR_SPECIAL;\r
 203                 ASCIIToHexDigit[i] = NON_HEX_DIGIT;\r
 204         }\r
 205         for(i = '0'; i <= '9'; i++)\r
 206         {\r
 207                 ASCIIToChrCode[i] = CHR_NUMBER;\r
 208                 ASCIIToHexDigit[i] = i-'0';\r
 209         }\r
 210         for(i = 'A'; i <= 'F'; i++)\r
 211         {\r
 212                 ASCIIToHexDigit[i] = 10+(i-'A');\r
 213         }\r
 214         for(i = 'a'; i <= 'f'; i++)\r
 215         {\r
 216                 ASCIIToHexDigit[i] = 10+(i-'a');\r
 217         }\r
 218         for(i = 'A'; i <= 'Z'; i++)\r
 219         {\r
 220                 ASCIIToChrCode[i] = CHR_LETTER;\r
 221         }\r
 222         for(i = 'a'; i <= 'z'; i++)\r
 223         {\r
 224                 ASCIIToChrCode[i] = CHR_LETTER;\r
 225         }\r
 226         ASCIIToChrCode[ASCII_QUOTE] = CHR_QUOTE;\r
 227         ASCIIToChrCode[ASCII_UNDERSCORE] = CHR_LETTER;\r
 228         ASCIIToChrCode[EOF_CHARACTER] = CHR_EOF;\r
 229         tk_String = TokenStringBuffer;\r
 230         IncLineNumber = FALSE;\r
 231         tk_IncludedLines = 0;\r
 232         SourceOpen = FALSE;\r
 233         *MasterSourceLine = '\0'; // master line - Ty 07jan2000\r
 234         MasterSourcePos = 0;      // master position - Ty 07jan2000\r
 235         ClearMasterSourceLine = TRUE; // clear the line to start\r
 236         qsort (Keywords, NUM_KEYWORDS, sizeof(Keywords[0]), SortKeywords);\r
 237         FileNames = MS_Alloc(4096, ERR_OUT_OF_MEMORY);\r
 238         FileNamesLen = 0;\r
 239         FileNamesMax = 4096;\r
 240 }\r
 241 \r
 242 //==========================================================================\r
 243 //\r
 244 // SortKeywords\r
 245 //\r
 246 //==========================================================================\r
 247 \r
 248 static int SortKeywords(const void *a, const void *b)\r
 249 {\r
 250         return strcmp (((struct keyword_s *)a)->name,\r
 251                 ((struct keyword_s *)b)->name);\r
 252 }\r
 253 \r
 254 //==========================================================================\r
 255 //\r
 256 // TK_OpenSource\r
 257 //\r
 258 //==========================================================================\r
 259 \r
 260 void TK_OpenSource(char *fileName)\r
 261 {\r
 262         int size;\r
 263 \r
 264         TK_CloseSource();\r
 265         size = MS_LoadFile(fileName, &FileStart);\r
 266         tk_SourceName = AddFileName(fileName);\r
 267         MakeIncludePath(fileName);\r
 268         SourceOpen = TRUE;\r
 269         FileEnd = FileStart+size;\r
 270         FilePtr = FileStart;\r
 271         tk_Line = 1;\r
 272         tk_Token = TK_NONE;\r
 273         AlreadyGot = FALSE;\r
 274         NestDepth = 0;\r
 275         NextChr();\r
 276 }\r
 277 \r
 278 //==========================================================================\r
 279 //\r
 280 // AddFileName\r
 281 //\r
 282 //==========================================================================\r
 283 \r
 284 static char *AddFileName(const char *name)\r
 285 {\r
 286         size_t len = strlen(name) + 1;\r
 287         char *namespot;\r
 288 \r
 289         if (FileNamesLen + len > FileNamesMax)\r
 290         {\r
 291                 FileNames = MS_Alloc(FileNamesMax, ERR_OUT_OF_MEMORY);\r
 292                 FileNamesLen = 0;\r
 293         }\r
 294         namespot = FileNames + FileNamesLen;\r
 295         memcpy(namespot, name, len);\r
 296         FileNamesLen += len;\r
 297         return namespot;\r
 298 }\r
 299 \r
 300 //==========================================================================\r
 301 //\r
 302 // MakeIncludePath\r
 303 //\r
 304 //==========================================================================\r
 305 \r
 306 static void MakeIncludePath(char *sourceName)\r
 307 {\r
 308         strcpy(IncludePath, sourceName);\r
 309         if(MS_StripFilename(IncludePath) == NO)\r
 310         {\r
 311                 IncludePath[0] = 0;\r
 312         }\r
 313         else\r
 314         { // Add a directory delimiter to the include path\r
 315                 strcat(IncludePath, DIRECTORY_DELIMITER);\r
 316         }\r
 317 }\r
 318 \r
 319 //==========================================================================\r
 320 //\r
 321 // TK_Include\r
 322 //\r
 323 //==========================================================================\r
 324 \r
 325 void TK_Include(char *fileName)\r
 326 {\r
 327         char sourceName[MAX_FILE_NAME_LENGTH];\r
 328         int size;\r
 329         nestInfo_t *info;\r
 330 \r
 331         MS_Message(MSG_DEBUG, "*Including %s\n", fileName);\r
 332         if(NestDepth == MAX_NESTED_SOURCES)\r
 333         {\r
 334                 ERR_Exit(ERR_INCL_NESTING_TOO_DEEP, YES, fileName);\r
 335         }\r
 336         info = &OpenFiles[NestDepth++];\r
 337         info->name = tk_SourceName;\r
 338         info->start = FileStart;\r
 339         info->end = FileEnd;\r
 340         info->position = FilePtr;\r
 341         info->line = tk_Line;\r
 342         info->incLineNumber = IncLineNumber;\r
 343         info->lastChar = Chr;\r
 344         info->imported = NO;\r
 345         strcpy(sourceName, IncludePath);\r
 346         strcat(sourceName, fileName);\r
 347         tk_SourceName = AddFileName(sourceName);\r
 348         size = MS_LoadFile(tk_SourceName, &FileStart);\r
 349         FileEnd = FileStart+size;\r
 350         FilePtr = FileStart;\r
 351         tk_Line = 1;\r
 352         IncLineNumber = FALSE;\r
 353         tk_Token = TK_NONE;\r
 354         AlreadyGot = FALSE;\r
 355         BumpMasterSourceLine('x',TRUE); // dummy x\r
 356         NextChr();\r
 357 }\r
 358 \r
 359 //==========================================================================\r
 360 //\r
 361 // TK_Import\r
 362 //\r
 363 //==========================================================================\r
 364 \r
 365 void TK_Import(char *fileName, enum ImportModes prevMode)\r
 366 {\r
 367         TK_Include (fileName);\r
 368         OpenFiles[NestDepth - 1].imported = YES;\r
 369         OpenFiles[NestDepth - 1].prevMode = prevMode;\r
 370         ImportMode = IMPORT_Importing;\r
 371 }\r
 372 \r
 373 //==========================================================================\r
 374 //\r
 375 // PopNestedSource\r
 376 //\r
 377 //==========================================================================\r
 378 \r
 379 static int PopNestedSource(enum ImportModes *prevMode)\r
 380 {\r
 381         nestInfo_t *info;\r
 382 \r
 383         MS_Message(MSG_DEBUG, "*Leaving %s\n", tk_SourceName);\r
 384         free(FileStart);\r
 385         SY_FreeConstants(NestDepth);\r
 386         tk_IncludedLines += tk_Line;\r
 387         info = &OpenFiles[--NestDepth];\r
 388         tk_SourceName = info->name;\r
 389         FileStart = info->start;\r
 390         FileEnd = info->end;\r
 391         FilePtr = info->position;\r
 392         tk_Line = info->line;\r
 393         IncLineNumber = info->incLineNumber;\r
 394         Chr = info->lastChar;\r
 395         tk_Token = TK_NONE;\r
 396         AlreadyGot = FALSE;\r
 397         *prevMode = info->prevMode;\r
 398         return info->imported ? 2 : 0;\r
 399 }\r
 400 \r
 401 //==========================================================================\r
 402 //\r
 403 // TK_CloseSource\r
 404 //\r
 405 //==========================================================================\r
 406 \r
 407 void TK_CloseSource(void)\r
 408 {\r
 409         int i;\r
 410 \r
 411         if(SourceOpen)\r
 412         {\r
 413                 free(FileStart);\r
 414                 for(i = 0; i < NestDepth; i++)\r
 415                 {\r
 416                         free(OpenFiles[i].start);\r
 417                 }\r
 418                 SourceOpen = FALSE;\r
 419         }\r
 420 }\r
 421 \r
 422 //==========================================================================\r
 423 //\r
 424 // TK_GetDepth\r
 425 //\r
 426 //==========================================================================\r
 427 \r
 428 int TK_GetDepth(void)\r
 429 {\r
 430         return NestDepth;\r
 431 }\r
 432 \r
 433 //==========================================================================\r
 434 //\r
 435 // TK_NextToken\r
 436 //\r
 437 //==========================================================================\r
 438 \r
 439 tokenType_t TK_NextToken(void)\r
 440 {\r
 441         enum ImportModes prevMode;\r
 442         boolean validToken;\r
 443 \r
 444         if(AlreadyGot == TRUE)\r
 445         {\r
 446                 int t = MasterSourcePos;\r
 447                 MasterSourcePos = PrevMasterSourcePos;\r
 448                 PrevMasterSourcePos = t;\r
 449                 AlreadyGot = FALSE;\r
 450                 return tk_Token;\r
 451         }\r
 452         validToken = NO;\r
 453         PrevMasterSourcePos = MasterSourcePos;\r
 454         do\r
 455         {\r
 456                 while(Chr == ASCII_SPACE)\r
 457                 {\r
 458                         NextChr();\r
 459                 }\r
 460                 switch(ASCIIToChrCode[(byte)Chr])\r
 461                 {\r
 462                 case CHR_EOF:\r
 463                         tk_Token = TK_EOF;\r
 464                         break;\r
 465                 case CHR_LETTER:\r
 466                         ProcessLetterToken();\r
 467                         break;\r
 468                 case CHR_NUMBER:\r
 469                         ProcessNumberToken();\r
 470                         break;\r
 471                 case CHR_QUOTE:\r
 472                         ProcessQuoteToken();\r
 473                         break;\r
 474                 default:\r
 475                         ProcessSpecialToken();\r
 476                         break;\r
 477                 }\r
 478                 if(tk_Token == TK_STARTCOMMENT)\r
 479                 {\r
 480                         SkipComment();\r
 481                 }\r
 482                 else if(tk_Token == TK_CPPCOMMENT)\r
 483                 {\r
 484                         SkipCPPComment();\r
 485                 }\r
 486                 else if((tk_Token == TK_EOF) && (NestDepth > 0))\r
 487                 {\r
 488                         if (PopNestedSource(&prevMode))\r
 489                         {\r
 490                                 ImportMode = prevMode;\r
 491                                 if(!ExporterFlagged)\r
 492                                 {\r
 493                                         ERR_Exit(ERR_EXPORTER_NOT_FLAGGED, NO);\r
 494                                 }\r
 495                                 SY_ClearShared();\r
 496                         }\r
 497                 }\r
 498                 else\r
 499                 {\r
 500                         validToken = YES;\r
 501                 }\r
 502         } while(validToken == NO);\r
 503         return tk_Token;\r
 504 }\r
 505 \r
 506 //==========================================================================\r
 507 //\r
 508 // TK_NextCharacter\r
 509 //\r
 510 //==========================================================================\r
 511 \r
 512 int TK_NextCharacter(void)\r
 513 {\r
 514         int c;\r
 515 \r
 516         while(Chr == ASCII_SPACE)\r
 517         {\r
 518                 NextChr();\r
 519         }\r
 520         c = (int)Chr;\r
 521         if(c == EOF_CHARACTER)\r
 522         {\r
 523                 c = -1;\r
 524         }\r
 525         NextChr();\r
 526         return c;\r
 527 }\r
 528 \r
 529 //==========================================================================\r
 530 //\r
 531 // TK_SkipPast\r
 532 //\r
 533 //==========================================================================\r
 534 \r
 535 void TK_SkipPast(tokenType_t token)\r
 536 {\r
 537         while (tk_Token != token)\r
 538         {\r
 539                 TK_NextToken();\r
 540         }\r
 541         TK_NextToken();\r
 542 }\r
 543 \r
 544 //==========================================================================\r
 545 //\r
 546 // TK_SkipTo\r
 547 //\r
 548 //==========================================================================\r
 549 \r
 550 void TK_SkipTo(tokenType_t token)\r
 551 {\r
 552         while (tk_Token != token)\r
 553         {\r
 554                 TK_NextToken();\r
 555         }\r
 556 }\r
 557 \r
 558 //==========================================================================\r
 559 //\r
 560 // TK_NextTokenMustBe\r
 561 //\r
 562 //==========================================================================\r
 563 \r
 564 boolean TK_NextTokenMustBe(tokenType_t token, error_t error)\r
 565 {\r
 566         if(TK_NextToken() != token)\r
 567         {\r
 568                 ERR_Error(error, YES);\r
 569                 /*\r
 570                 if(skipToken == TK_EOF)\r
 571                 {\r
 572                         ERR_Finish();\r
 573                 }\r
 574                 else if(skipToken != TK_NONE)\r
 575                 {\r
 576                         TK_SkipPast(skipToken);\r
 577                 }\r
 578                 return NO;\r
 579                 */\r
 580                 ERR_Finish();\r
 581         }\r
 582         return YES;\r
 583 }\r
 584 \r
 585 //==========================================================================\r
 586 //\r
 587 // TK_TokenMustBe\r
 588 //\r
 589 //==========================================================================\r
 590 \r
 591 boolean TK_TokenMustBe(tokenType_t token, error_t error)\r
 592 {\r
 593         if (token == TK_SEMICOLON && forSemicolonHack)\r
 594         {\r
 595                 token = TK_RPAREN;\r
 596         }\r
 597         if(tk_Token != token)\r
 598         {\r
 599                 ERR_Error(error, YES);\r
 600                 /*\r
 601                 if(skipToken == TK_EOF)\r
 602                 {\r
 603                         ERR_Finish();\r
 604                 }\r
 605                 else if(skipToken != TK_NONE)\r
 606                 {\r
 607                         while(tk_Token != skipToken)\r
 608                         {\r
 609                                 TK_NextToken();\r
 610                         }\r
 611                 }\r
 612                 return NO;\r
 613                 */\r
 614                 ERR_Finish();\r
 615         }\r
 616         return YES;\r
 617 }\r
 618 \r
 619 //==========================================================================\r
 620 //\r
 621 // TK_Member\r
 622 //\r
 623 //==========================================================================\r
 624 \r
 625 boolean TK_Member(tokenType_t *list)\r
 626 {\r
 627         int i;\r
 628 \r
 629         for(i = 0; list[i] != TK_NONE; i++)\r
 630         {\r
 631                 if(tk_Token == list[i])\r
 632                 {\r
 633                         return YES;\r
 634                 }\r
 635         }\r
 636         return NO;\r
 637 }\r
 638 \r
 639 //==========================================================================\r
 640 //\r
 641 // TK_Undo\r
 642 //\r
 643 //==========================================================================\r
 644 \r
 645 void TK_Undo(void)\r
 646 {\r
 647         if(tk_Token != TK_NONE)\r
 648         {\r
 649                 if (AlreadyGot == FALSE)\r
 650                 {\r
 651                         int t = MasterSourcePos;\r
 652                         MasterSourcePos = PrevMasterSourcePos;\r
 653                         PrevMasterSourcePos = t;\r
 654                         AlreadyGot = TRUE;\r
 655                 }\r
 656         }\r
 657 }\r
 658 \r
 659 //==========================================================================\r
 660 //\r
 661 // ProcessLetterToken\r
 662 //\r
 663 //==========================================================================\r
 664 \r
 665 static void ProcessLetterToken(void)\r
 666 {\r
 667         int i;\r
 668         char *text;\r
 669 \r
 670         i = 0;\r
 671         text = TokenStringBuffer;\r
 672         while (ASCIIToChrCode[(byte)Chr] == CHR_LETTER\r
 673                 || ASCIIToChrCode[(byte)Chr] == CHR_NUMBER)\r
 674         {\r
 675                 if(++i == MAX_IDENTIFIER_LENGTH)\r
 676                 {\r
 677                         ERR_Error(ERR_IDENTIFIER_TOO_LONG, YES);\r
 678                 }\r
 679                 if(i < MAX_IDENTIFIER_LENGTH)\r
 680                 {\r
 681                         *text++ = Chr;\r
 682                 }\r
 683                 NextChr();\r
 684         }\r
 685         *text = 0;\r
 686         MS_StrLwr(TokenStringBuffer);\r
 687         if(CheckForKeyword() == FALSE\r
 688                 && CheckForLineSpecial() == FALSE\r
 689                 && CheckForConstant() == FALSE)\r
 690         {\r
 691                 tk_Token = TK_IDENTIFIER;\r
 692         }\r
 693 }\r
 694 \r
 695 //==========================================================================\r
 696 //\r
 697 // CheckForKeyword\r
 698 //\r
 699 //==========================================================================\r
 700 \r
 701 static boolean CheckForKeyword(void)\r
 702 {\r
 703         int min, max, probe, lexx;\r
 704 \r
 705         // [RH] Use a binary search\r
 706         min = 0;\r
 707         max = NUM_KEYWORDS-1;\r
 708         probe = NUM_KEYWORDS/2;\r
 709 \r
 710         while (max - min >= 0)\r
 711         {\r
 712                 lexx = strcmp(tk_String, Keywords[probe].name);\r
 713                 if(lexx == 0)\r
 714                 {\r
 715                         tk_Token = Keywords[probe].token;\r
 716                         return TRUE;\r
 717                 }\r
 718                 else if(lexx < 0)\r
 719                 {\r
 720                         max = probe-1;\r
 721                 }\r
 722                 else\r
 723                 {\r
 724                         min = probe+1;\r
 725                 }\r
 726                 probe = (max-min)/2+min;\r
 727         }\r
 728         return FALSE;\r
 729 }\r
 730 \r
 731 //==========================================================================\r
 732 //\r
 733 // CheckForLineSpecial\r
 734 //\r
 735 //==========================================================================\r
 736 \r
 737 static boolean CheckForLineSpecial(void)\r
 738 {\r
 739         symbolNode_t *sym;\r
 740 \r
 741         sym = SY_FindGlobal(tk_String);\r
 742         if(sym == NULL)\r
 743         {\r
 744                 return FALSE;\r
 745         }\r
 746         if(sym->type != SY_SPECIAL)\r
 747         {\r
 748                 return FALSE;\r
 749         }\r
 750         tk_Token = TK_LINESPECIAL;\r
 751         tk_SpecialValue = sym->info.special.value;\r
 752         tk_SpecialArgCount = sym->info.special.argCount;\r
 753         return TRUE;\r
 754 }\r
 755 \r
 756 //==========================================================================\r
 757 //\r
 758 // CheckForConstant\r
 759 //\r
 760 //==========================================================================\r
 761 \r
 762 static boolean CheckForConstant(void)\r
 763 {\r
 764         symbolNode_t *sym;\r
 765 \r
 766         sym = SY_FindGlobal(tk_String);\r
 767         if(sym == NULL)\r
 768         {\r
 769                 return FALSE;\r
 770         }\r
 771         if(sym->type != SY_CONSTANT)\r
 772         {\r
 773                 return FALSE;\r
 774         }\r
 775         tk_Token = TK_NUMBER;\r
 776         tk_Number = sym->info.constant.value;\r
 777         return TRUE;\r
 778 }\r
 779 \r
 780 //==========================================================================\r
 781 //\r
 782 // ProcessNumberToken\r
 783 //\r
 784 //==========================================================================\r
 785 \r
 786 static void ProcessNumberToken(void)\r
 787 {\r
 788         char c;\r
 789 \r
 790         c = Chr;\r
 791         NextChr();\r
 792         if(c == '0' && (Chr == 'x' || Chr == 'X'))\r
 793         { // Hexadecimal constant\r
 794                 NextChr();\r
 795                 EvalHexConstant();\r
 796                 return;\r
 797         }\r
 798         tk_Number = c-'0';\r
 799         while(ASCIIToChrCode[(byte)Chr] == CHR_NUMBER)\r
 800         {\r
 801                 tk_Number = 10*tk_Number+(Chr-'0');\r
 802                 NextChr();\r
 803         }\r
 804         if(Chr == '.')\r
 805         { // Fixed point\r
 806                 NextChr(); // Skip period\r
 807                 EvalFixedConstant(tk_Number);\r
 808                 return;\r
 809         }\r
 810         if(Chr == ASCII_UNDERSCORE)\r
 811         {\r
 812                 NextChr(); // Skip underscore\r
 813                 EvalRadixConstant();\r
 814                 return;\r
 815         }\r
 816         tk_Token = TK_NUMBER;\r
 817 }\r
 818 \r
 819 //==========================================================================\r
 820 //\r
 821 // EvalFixedConstant\r
 822 //\r
 823 //==========================================================================\r
 824 \r
 825 static void EvalFixedConstant(int whole)\r
 826 {\r
 827         int frac;\r
 828         int divisor;\r
 829 \r
 830         frac = 0;\r
 831         divisor = 1;\r
 832         while(ASCIIToChrCode[(byte)Chr] == CHR_NUMBER)\r
 833         {\r
 834                 frac = 10*frac+(Chr-'0');\r
 835                 divisor *= 10;\r
 836                 NextChr();\r
 837         }\r
 838         tk_Number = (whole<<16)+((frac<<16)/divisor);\r
 839         tk_Token = TK_NUMBER;\r
 840 }\r
 841 \r
 842 //==========================================================================\r
 843 //\r
 844 // EvalHexConstant\r
 845 //\r
 846 //==========================================================================\r
 847 \r
 848 static void EvalHexConstant(void)\r
 849 {\r
 850         tk_Number = 0;\r
 851         while(ASCIIToHexDigit[(byte)Chr] != NON_HEX_DIGIT)\r
 852         {\r
 853                 tk_Number = (tk_Number<<4)+ASCIIToHexDigit[(byte)Chr];\r
 854                 NextChr();\r
 855         }\r
 856         tk_Token = TK_NUMBER;\r
 857 }\r
 858 \r
 859 //==========================================================================\r
 860 //\r
 861 // EvalRadixConstant\r
 862 //\r
 863 //==========================================================================\r
 864 \r
 865 static void EvalRadixConstant(void)\r
 866 {\r
 867         int radix;\r
 868         int digitVal;\r
 869 \r
 870         radix = tk_Number;\r
 871         if(radix < 2 || radix > 36)\r
 872         {\r
 873                 ERR_Error(ERR_BAD_RADIX_CONSTANT, YES, NULL);\r
 874                 radix = 36;\r
 875         }\r
 876         tk_Number = 0;\r
 877         while((digitVal = DigitValue(Chr, radix)) != -1)\r
 878         {\r
 879                 tk_Number = radix*tk_Number+digitVal;\r
 880                 NextChr();\r
 881         }\r
 882         tk_Token = TK_NUMBER;\r
 883 }\r
 884 \r
 885 //==========================================================================\r
 886 //\r
 887 // DigitValue\r
 888 //\r
 889 // Returns -1 if the digit is not allowed in the specified radix.\r
 890 //\r
 891 //==========================================================================\r
 892 \r
 893 static int DigitValue(char digit, int radix)\r
 894 {\r
 895         digit = toupper(digit);\r
 896         if(digit < '0' || (digit > '9' && digit < 'A') || digit > 'Z')\r
 897         {\r
 898                 return -1;\r
 899         }\r
 900         if(digit > '9')\r
 901         {\r
 902                 digit = 10+digit-'A';\r
 903         }\r
 904         else\r
 905         {\r
 906                 digit -= '0';\r
 907         }\r
 908         if(digit >= radix)\r
 909         {\r
 910                 return -1;\r
 911         }\r
 912         return digit;\r
 913 }\r
 914 \r
 915 //==========================================================================\r
 916 //\r
 917 // ProcessQuoteToken\r
 918 //\r
 919 //==========================================================================\r
 920 \r
 921 static void ProcessQuoteToken(void)\r
 922 {\r
 923         int i;\r
 924         char *text;\r
 925         boolean escaped;\r
 926 \r
 927         i = 0;\r
 928         escaped = FALSE;\r
 929         text = TokenStringBuffer;\r
 930         NextChr();\r
 931         while(Chr != EOF_CHARACTER)\r
 932         {\r
 933                 if(Chr == ASCII_QUOTE && escaped == 0) // [JB]\r
 934                 {\r
 935                         break;\r
 936                 }\r
 937                 if(++i == MAX_QUOTED_LENGTH)\r
 938                 {\r
 939                         ERR_Error(ERR_STRING_TOO_LONG, YES, NULL);\r
 940                 }\r
 941                 if(i < MAX_QUOTED_LENGTH)\r
 942                 {\r
 943                         *text++ = Chr;\r
 944                 }\r
 945                 // escape the character after a backslash [JB]\r
 946                 if(Chr == ASCII_BACKSLASH)\r
 947                         escaped ^= (Chr == ASCII_BACKSLASH);\r
 948                 else\r
 949                         escaped = FALSE;\r
 950                 NextChr();\r
 951         }\r
 952         *text = 0;\r
 953         if(Chr == ASCII_QUOTE)\r
 954         {\r
 955                 NextChr();\r
 956         }\r
 957         tk_Token = TK_STRING;\r
 958 }\r
 959 \r
 960 //==========================================================================\r
 961 //\r
 962 // ProcessSpecialToken\r
 963 //\r
 964 //==========================================================================\r
 965 \r
 966 static void ProcessSpecialToken(void)\r
 967 {\r
 968         char c;\r
 969 \r
 970         c = Chr;\r
 971         NextChr();\r
 972         switch(c)\r
 973         {\r
 974                 case '+':\r
 975                         switch(Chr)\r
 976                         {\r
 977                                 case '=':\r
 978                                         tk_Token = TK_ADDASSIGN;\r
 979                                         NextChr();\r
 980                                         break;\r
 981                                 case '+':\r
 982                                         tk_Token = TK_INC;\r
 983                                         NextChr();\r
 984                                         break;\r
 985                                 default:\r
 986                                         tk_Token = TK_PLUS;\r
 987                                         break;\r
 988                         }\r
 989                         break;\r
 990                 case '-':\r
 991                         switch(Chr)\r
 992                         {\r
 993                                 case '=':\r
 994                                         tk_Token = TK_SUBASSIGN;\r
 995                                         NextChr();\r
 996                                         break;\r
 997                                 case '-':\r
 998                                         tk_Token = TK_DEC;\r
 999                                         NextChr();\r
1000                                         break;\r
1001                                 default:\r
1002                                         tk_Token = TK_MINUS;\r
1003                                         break;\r
1004                         }\r
1005                         break;\r
1006                 case '*':\r
1007                         switch(Chr)\r
1008                         {\r
1009                                 case '=':\r
1010                                         tk_Token = TK_MULASSIGN;\r
1011                                         NextChr();\r
1012                                         break;\r
1013                                 case '/':\r
1014                                         tk_Token = TK_ENDCOMMENT;\r
1015                                         NextChr();\r
1016                                         break;\r
1017                                 default:\r
1018                                         tk_Token = TK_ASTERISK;\r
1019                                         break;\r
1020                         }\r
1021                         break;\r
1022                 case '/':\r
1023                         switch(Chr)\r
1024                         {\r
1025                                 case '=':\r
1026                                         tk_Token = TK_DIVASSIGN;\r
1027                                         NextChr();\r
1028                                         break;\r
1029                                 case '/':\r
1030                                         tk_Token = TK_CPPCOMMENT;\r
1031                                         break;\r
1032                                 case '*':\r
1033                                         tk_Token = TK_STARTCOMMENT;\r
1034                                         NextChr();\r
1035                                         break;\r
1036                                 default:\r
1037                                         tk_Token = TK_SLASH;\r
1038                                         break;\r
1039                         }\r
1040                         break;\r
1041                 case '%':\r
1042                         if(Chr == '=')\r
1043                         {\r
1044                                 tk_Token = TK_MODASSIGN;\r
1045                                 NextChr();\r
1046                         }\r
1047                         else\r
1048                         {\r
1049                                 tk_Token = TK_PERCENT;\r
1050                         }\r
1051                         break;\r
1052                 case '=':\r
1053                         if(Chr == '=')\r
1054                         {\r
1055                                 tk_Token = TK_EQ;\r
1056                                 NextChr();\r
1057                         }\r
1058                         else\r
1059                         {\r
1060                                 tk_Token = TK_ASSIGN;\r
1061                         }\r
1062                         break;\r
1063                 case '<':\r
1064                         if(Chr == '=')\r
1065                         {\r
1066                                 tk_Token = TK_LE;\r
1067                                 NextChr();\r
1068                         }\r
1069                         else if(Chr == '<')\r
1070                         {\r
1071                                 NextChr();\r
1072                                 if(Chr == '=')\r
1073                                 {\r
1074                                         tk_Token = TK_LSASSIGN;\r
1075                                         NextChr();\r
1076                                 }\r
1077                                 else\r
1078                                 {\r
1079                                         tk_Token = TK_LSHIFT;\r
1080                                 }\r
1081                                 \r
1082                         }\r
1083                         else\r
1084                         {\r
1085                                 tk_Token = TK_LT;\r
1086                         }\r
1087                         break;\r
1088                 case '>':\r
1089                         if(Chr == '=')\r
1090                         {\r
1091                                 tk_Token = TK_GE;\r
1092                                 NextChr();\r
1093                         }\r
1094                         else if(Chr == '>')\r
1095                         {\r
1096                                 NextChr();\r
1097                                 if(Chr == '=')\r
1098                                 {\r
1099                                         tk_Token = TK_RSASSIGN;\r
1100                                         NextChr();\r
1101                                 }\r
1102                                 else\r
1103                                 {\r
1104                                         tk_Token = TK_RSHIFT;\r
1105                                 }\r
1106                         }\r
1107                         else\r
1108                         {\r
1109                                 tk_Token = TK_GT;\r
1110                         }\r
1111                         break;\r
1112                 case '!':\r
1113                         if(Chr == '=')\r
1114                         {\r
1115                                 tk_Token = TK_NE;\r
1116                                 NextChr();\r
1117                         }\r
1118                         else\r
1119                         {\r
1120                                 tk_Token = TK_NOT;\r
1121                         }\r
1122                         break;\r
1123                 case '&':\r
1124                         if(Chr == '&')\r
1125                         {\r
1126                                 tk_Token = TK_ANDLOGICAL;\r
1127                                 NextChr();\r
1128                         }\r
1129                         else if(Chr == '=')\r
1130                         {\r
1131                                 tk_Token = TK_ANDASSIGN;\r
1132                                 NextChr();\r
1133                         }\r
1134                         else\r
1135                         {\r
1136                                 tk_Token = TK_ANDBITWISE;\r
1137                         }\r
1138                         break;\r
1139                 case '|':\r
1140                         if(Chr == '|')\r
1141                         {\r
1142                                 tk_Token = TK_ORLOGICAL;\r
1143                                 NextChr();\r
1144                         }\r
1145                         else if(Chr == '=')\r
1146                         {\r
1147                                 tk_Token = TK_ORASSIGN;\r
1148                                 NextChr();\r
1149                         }\r
1150                         else\r
1151                         {\r
1152                                 tk_Token = TK_ORBITWISE;\r
1153                         }\r
1154                         break;\r
1155                 case '(':\r
1156                         tk_Token = TK_LPAREN;\r
1157                         break;\r
1158                 case ')':\r
1159                         tk_Token = TK_RPAREN;\r
1160                         break;\r
1161                 case '{':\r
1162                         tk_Token = TK_LBRACE;\r
1163                         break;\r
1164                 case '}':\r
1165                         tk_Token = TK_RBRACE;\r
1166                         break;\r
1167                 case '[':\r
1168                         tk_Token = TK_LBRACKET;\r
1169                         break;\r
1170                 case ']':\r
1171                         tk_Token = TK_RBRACKET;\r
1172                         break;\r
1173                 case ':':\r
1174                         tk_Token = TK_COLON;\r
1175                         break;\r
1176                 case ';':\r
1177                         tk_Token = TK_SEMICOLON;\r
1178                         break;\r
1179                 case ',':\r
1180                         tk_Token = TK_COMMA;\r
1181                         break;\r
1182                 case '.':\r
1183                         tk_Token = TK_PERIOD;\r
1184                         break;\r
1185                 case '#':\r
1186                         tk_Token = TK_NUMBERSIGN;\r
1187                         break;\r
1188                 case '^':\r
1189                         if(Chr == '=')\r
1190                         {\r
1191                                 tk_Token = TK_EORASSIGN;\r
1192                                 NextChr();\r
1193                         }\r
1194                         else\r
1195                         {\r
1196                                 tk_Token = TK_EORBITWISE;\r
1197                         }\r
1198                         break;\r
1199                 case '~':\r
1200                         tk_Token = TK_TILDE;\r
1201                         break;\r
1202                 case '\'':\r
1203                         if(Chr == '\\')\r
1204                         {\r
1205                                 NextChr();\r
1206                                 switch(Chr)\r
1207                                 {\r
1208                                 case '0': case '1': case '2': case '3':\r
1209                                 case '4': case '5': case '6': case '7':\r
1210                                         tk_Number = OctalChar();\r
1211                                         break;\r
1212                                 case 'x': case 'X':\r
1213                                         NextChr();\r
1214                                         EvalHexConstant();\r
1215                                         if(Chr != '\'')\r
1216                                         {\r
1217                                                 ERR_Exit(ERR_BAD_CHARACTER_CONSTANT, YES, NULL);\r
1218                                         }\r
1219                                         NextChr();\r
1220                                         break;\r
1221                                 case 'a':\r
1222                                         tk_Number = '\a';\r
1223                                         break;\r
1224                                 case 'b':\r
1225                                         tk_Number = '\b';\r
1226                                         break;\r
1227                                 case 't':\r
1228                                         tk_Number = '\t';\r
1229                                         break;\r
1230                                 case 'v':\r
1231                                         tk_Number = '\v';\r
1232                                         break;\r
1233                                 case 'n':\r
1234                                         tk_Number = '\n';\r
1235                                         break;\r
1236                                 case 'f':\r
1237                                         tk_Number = '\f';\r
1238                                         break;\r
1239                                 case 'r':\r
1240                                         tk_Number = '\r';\r
1241                                         break;\r
1242                                 case '\'':\r
1243                                         tk_Number = Chr;\r
1244                                         break;\r
1245                                 default:\r
1246                                         ERR_Exit(ERR_BAD_CHARACTER_CONSTANT, YES, NULL);\r
1247                                 }\r
1248                                 tk_Token = TK_NUMBER;\r
1249                         }\r
1250                         else if(Chr == '\'')\r
1251                         {\r
1252                                 ERR_Exit(ERR_BAD_CHARACTER_CONSTANT, YES, NULL);\r
1253                         }\r
1254                         else\r
1255                         {\r
1256                                 tk_Number = Chr;\r
1257                                 tk_Token = TK_NUMBER;\r
1258                         }\r
1259                         NextChr();\r
1260                         if(Chr != '\'')\r
1261                         {\r
1262                                 ERR_Exit(ERR_BAD_CHARACTER_CONSTANT, YES, NULL);\r
1263                         }\r
1264                         NextChr();\r
1265                         break;\r
1266                 default:\r
1267                         ERR_Exit(ERR_BAD_CHARACTER, YES, NULL);\r
1268                         break;\r
1269         }\r
1270 }\r
1271 \r
1272 //==========================================================================\r
1273 //\r
1274 // NextChr\r
1275 //\r
1276 //==========================================================================\r
1277 \r
1278 static void NextChr(void)\r
1279 {\r
1280         if(FilePtr >= FileEnd)\r
1281         {\r
1282                 Chr = EOF_CHARACTER;\r
1283                 return;\r
1284         }\r
1285         if(IncLineNumber == TRUE)\r
1286         {\r
1287                 tk_Line++;\r
1288                 IncLineNumber = FALSE;\r
1289                 BumpMasterSourceLine('x',TRUE); // dummy x\r
1290         }\r
1291         Chr = *FilePtr++;\r
1292         if(Chr < ASCII_SPACE && Chr >= 0)       // Allow high ASCII characters\r
1293         {\r
1294                 if(Chr == '\n')\r
1295                 {\r
1296                         IncLineNumber = TRUE;\r
1297                 }\r
1298                 Chr = ASCII_SPACE;\r
1299         }\r
1300         BumpMasterSourceLine(Chr,FALSE);\r
1301 }\r
1302 \r
1303 //==========================================================================\r
1304 //\r
1305 // PeekChr // [JB]\r
1306 //\r
1307 //==========================================================================\r
1308 \r
1309 static int PeekChr(void)\r
1310 {\r
1311         char ch;\r
1312         if(FilePtr >= FileEnd)\r
1313         {\r
1314                 return EOF_CHARACTER;\r
1315         }\r
1316         ch = *FilePtr-1;\r
1317         if(ch < ASCII_SPACE && ch >= 0) // Allow high ASCII characters\r
1318         {\r
1319                 ch = ASCII_SPACE;\r
1320         }\r
1321         return ch;\r
1322 }\r
1323 \r
1324 //==========================================================================\r
1325 //\r
1326 // OctalChar // [JB]\r
1327 //\r
1328 //==========================================================================\r
1329 \r
1330 static int OctalChar() \r
1331 {\r
1332         int digits = 1;\r
1333         int code = Chr - '0';\r
1334         while(digits < 4 && PeekChr() >= '0' && PeekChr() <= '7')\r
1335         {\r
1336                 NextChr();\r
1337                 code = (code << 3) + Chr - '0';\r
1338         }\r
1339         return code;\r
1340 }\r
1341 \r
1342 //==========================================================================\r
1343 //\r
1344 // SkipComment\r
1345 //\r
1346 //==========================================================================\r
1347 \r
1348 void SkipComment(void)\r
1349 {\r
1350         boolean first;\r
1351 \r
1352         first = FALSE;\r
1353         while(Chr != EOF_CHARACTER)\r
1354         {\r
1355                 if(first == TRUE && Chr == '/')\r
1356                 {\r
1357                         break;\r
1358                 }\r
1359                 first = (Chr == '*');\r
1360                 NextChr();\r
1361         }\r
1362         NextChr();\r
1363 }\r
1364 \r
1365 //==========================================================================\r
1366 //\r
1367 // SkipCPPComment\r
1368 //\r
1369 //==========================================================================\r
1370 \r
1371 void SkipCPPComment(void)\r
1372 {\r
1373         while(FilePtr < FileEnd)\r
1374         {\r
1375                 if(*FilePtr++ == '\n')\r
1376                 {\r
1377                         tk_Line++;\r
1378                         BumpMasterSourceLine('x',TRUE); // dummy x\r
1379                         break;\r
1380                 }\r
1381         }\r
1382         NextChr();\r
1383 }\r
1384 \r
1385 //==========================================================================\r
1386 //\r
1387 // BumpMasterSourceLine\r
1388 //\r
1389 //==========================================================================\r
1390 \r
1391 void BumpMasterSourceLine(char Chr, boolean clear) // master line - Ty 07jan2000\r
1392 {\r
1393         if (ClearMasterSourceLine)  // set to clear last time, clear now for first character\r
1394         {\r
1395                 *MasterSourceLine = '\0';\r
1396                 MasterSourcePos = 0;\r
1397                 ClearMasterSourceLine = FALSE;\r
1398         }\r
1399         if (clear)\r
1400         {\r
1401                 ClearMasterSourceLine = TRUE;\r
1402         }\r
1403         else\r
1404         {\r
1405                 if (MasterSourcePos < MAX_STATEMENT_LENGTH)\r
1406                         MasterSourceLine[MasterSourcePos++] = Chr;\r
1407         }\r
1408 }\r
1409 \r
1410 //==========================================================================\r
1411 //\r
1412 // TK_SkipLine\r
1413 //\r
1414 //==========================================================================\r
1415 \r
1416 void TK_SkipLine(void)\r
1417 {\r
1418         char *sourcenow = tk_SourceName;\r
1419         int linenow = tk_Line;\r
1420         do TK_NextToken(); while (tk_Line == linenow && tk_SourceName == sourcenow);\r
1421 }\r