Preprocessor: 1) Rationalize the "fixed atom" scheme, 2) remove redundant lookups...

This simplification is a prelude to eliminating what I appear unnecessary
symbol inserts into tables when tokenizing in the preprecessor, which
show up as taking notable time. (Performance issue.)  It also simply makes
the preprocessor easier to understand, which it is badly in need of.

diff --git a/glslang/Include/Common.h b/glslang/Include/Common.h
index f57a36b..3580a6d 100644 (file)
--- a/glslang/Include/Common.h
+++ b/glslang/Include/Common.h
@@ -92,6 +92,7 @@ namespace glslang {
 typedef pool_allocator<char> TStringAllocator;
 typedef std::basic_string <char, std::char_traits<char>, TStringAllocator> TString;
 
+// Repackage the std::hash for use by unordered map/set with a TString key.
 struct TStringHash {
     size_t operator()(const TString& string) const { return std::hash<TString>()(string); }
 };

diff --git a/glslang/MachineIndependent/ParseHelper.h b/glslang/MachineIndependent/ParseHelper.h
index 0680f27..d3a1e11 100644 (file)
--- a/glslang/MachineIndependent/ParseHelper.h
+++ b/glslang/MachineIndependent/ParseHelper.h
@@ -306,7 +306,7 @@ protected:
     TInputScanner* currentScanner;
     int numErrors;               // number of compile-time errors encountered
     bool parsingBuiltins;        // true if parsing built-in symbols/functions
-    std::unordered_map<TString, TExtensionBehavior> extensionBehavior;    // for each extension string, what its current behavior is set to
+    TUnorderedMap<TString, TExtensionBehavior> extensionBehavior;    // for each extension string, what its current behavior is set to
     static const int maxSamplerIndex = EsdNumDims * (EbtNumTypes * (2 * 2 * 2)); // see computeSamplerTypeIndex()
     TPrecisionQualifier defaultSamplerPrecision[maxSamplerIndex];
     bool afterEOF;

diff --git a/glslang/MachineIndependent/Scan.cpp b/glslang/MachineIndependent/Scan.cpp
index 8b22cdb..2cb965f 100644 (file)
--- a/glslang/MachineIndependent/Scan.cpp
+++ b/glslang/MachineIndependent/Scan.cpp
@@ -567,33 +567,38 @@ int TScanContext::tokenize(TPpContext* pp, TParserToken& token)
             parseContext.error(loc, "illegal use of escape character", "\\", "");
             break;
 
-        case CPP_AND_OP:                return AND_OP;
-        case CPP_SUB_ASSIGN:            return SUB_ASSIGN;
-        case CPP_MOD_ASSIGN:            return MOD_ASSIGN;
-        case CPP_ADD_ASSIGN:            return ADD_ASSIGN;
-        case CPP_DIV_ASSIGN:            return DIV_ASSIGN;
-        case CPP_MUL_ASSIGN:            return MUL_ASSIGN;
-        case CPP_EQ_OP:                 return EQ_OP;
-        case CPP_XOR_OP:                return XOR_OP;
-        case CPP_GE_OP:                 return GE_OP;
-        case CPP_RIGHT_OP:              return RIGHT_OP;
-        case CPP_LE_OP:                 return LE_OP;
-        case CPP_LEFT_OP:               return LEFT_OP;
-        case CPP_DEC_OP:                return DEC_OP;
-        case CPP_NE_OP:                 return NE_OP;
-        case CPP_OR_OP:                 return OR_OP;
-        case CPP_INC_OP:                return INC_OP;
-        case CPP_RIGHT_ASSIGN:          return RIGHT_ASSIGN;
-        case CPP_LEFT_ASSIGN:           return LEFT_ASSIGN;
-        case CPP_AND_ASSIGN:            return AND_ASSIGN;
-        case CPP_OR_ASSIGN:             return OR_ASSIGN;
-        case CPP_XOR_ASSIGN:            return XOR_ASSIGN;
+        case PpAtomAdd:                return ADD_ASSIGN;
+        case PpAtomSub:                return SUB_ASSIGN;
+        case PpAtomMul:                return MUL_ASSIGN;
+        case PpAtomDiv:                return DIV_ASSIGN;
+        case PpAtomMod:                return MOD_ASSIGN;
+
+        case PpAtomRight:              return RIGHT_OP;
+        case PpAtomLeft:               return LEFT_OP;
+
+        case PpAtomRightAssign:        return RIGHT_ASSIGN;
+        case PpAtomLeftAssign:         return LEFT_ASSIGN;
+        case PpAtomAndAssign:          return AND_ASSIGN;
+        case PpAtomOrAssign:           return OR_ASSIGN;
+        case PpAtomXorAssign:          return XOR_ASSIGN;
+
+        case PpAtomAnd:                return AND_OP;
+        case PpAtomOr:                 return OR_OP;
+        case PpAtomXor:                return XOR_OP;
+
+        case PpAtomEQ:                 return EQ_OP;
+        case PpAtomGE:                 return GE_OP;
+        case PpAtomNE:                 return NE_OP;
+        case PpAtomLE:                 return LE_OP;
+
+        case PpAtomDecrement:          return DEC_OP;
+        case PpAtomIncrement:          return INC_OP;
                                    
-        case CPP_INTCONSTANT:           parserToken->sType.lex.i = ppToken.ival;       return INTCONSTANT;
-        case CPP_UINTCONSTANT:          parserToken->sType.lex.i = ppToken.ival;       return UINTCONSTANT;
-        case CPP_FLOATCONSTANT:         parserToken->sType.lex.d = ppToken.dval;       return FLOATCONSTANT;
-        case CPP_DOUBLECONSTANT:        parserToken->sType.lex.d = ppToken.dval;       return DOUBLECONSTANT;
-        case CPP_IDENTIFIER:            return tokenizeIdentifier();
+        case PpAtomConstInt:           parserToken->sType.lex.i = ppToken.ival;       return INTCONSTANT;
+        case PpAtomConstUint:          parserToken->sType.lex.i = ppToken.ival;       return UINTCONSTANT;
+        case PpAtomConstFloat:         parserToken->sType.lex.d = ppToken.dval;       return FLOATCONSTANT;
+        case PpAtomConstDouble:        parserToken->sType.lex.d = ppToken.dval;       return DOUBLECONSTANT;
+        case PpAtomIdentifier:         return tokenizeIdentifier();
 
         case EOF:                       return 0;
                                    

diff --git a/glslang/MachineIndependent/preprocessor/Pp.cpp b/glslang/MachineIndependent/preprocessor/Pp.cpp
index 0e035c2..e246a6c 100644 (file)
--- a/glslang/MachineIndependent/preprocessor/Pp.cpp
+++ b/glslang/MachineIndependent/preprocessor/Pp.cpp
@@ -94,32 +94,6 @@ namespace glslang {
 
 int TPpContext::InitCPP()
 {
-    // Add various atoms needed by the CPP line scanner:
-    bindAtom = LookUpAddString("bind");
-    constAtom = LookUpAddString("const");
-    defaultAtom = LookUpAddString("default");
-    defineAtom = LookUpAddString("define");
-    definedAtom = LookUpAddString("defined");
-    elifAtom = LookUpAddString("elif");
-    elseAtom = LookUpAddString("else");
-    endifAtom = LookUpAddString("endif");
-    ifAtom = LookUpAddString("if");
-    ifdefAtom = LookUpAddString("ifdef");
-    ifndefAtom = LookUpAddString("ifndef");
-    includeAtom = LookUpAddString("include");
-    lineAtom = LookUpAddString("line");
-    pragmaAtom = LookUpAddString("pragma");
-    texunitAtom = LookUpAddString("texunit");
-    undefAtom = LookUpAddString("undef");
-    errorAtom = LookUpAddString("error");
-    __LINE__Atom = LookUpAddString("__LINE__");
-    __FILE__Atom = LookUpAddString("__FILE__");
-    __VERSION__Atom = LookUpAddString("__VERSION__");
-    versionAtom = LookUpAddString("version");
-    coreAtom = LookUpAddString("core");
-    compatibilityAtom = LookUpAddString("compatibility");
-    esAtom = LookUpAddString("es");
-    extensionAtom = LookUpAddString("extension");
     pool = mem_CreatePool(0, 0);
 
     return 1;
@@ -133,17 +107,18 @@ int TPpContext::CPPdefine(TPpToken* ppToken)
 
     // get macro name
     int token = scanToken(ppToken);
-    if (token != CPP_IDENTIFIER) {
+    if (token != PpAtomIdentifier) {
         parseContext.ppError(ppToken->loc, "must be followed by macro name", "#define", "");
         return token;
     }
-    int atom = ppToken->atom;
-    const char* definedName = GetAtomString(atom);
     if (ppToken->loc.string >= 0) {
         // We are in user code; check for reserved name use:
-        parseContext.reservedPpErrorCheck(ppToken->loc, definedName, "#define");
+        parseContext.reservedPpErrorCheck(ppToken->loc, ppToken->name, "#define");
     }
 
+    // save the original atom
+    const int defAtom = ppToken->atom;
+
     // gather parameters to the macro, between (...)
     token = scanToken(ppToken);
     if (token == '(' && ! ppToken->space) {
@@ -153,7 +128,7 @@ int TPpContext::CPPdefine(TPpToken* ppToken)
             token = scanToken(ppToken);
             if (argc == 0 && token == ')') 
                 break;
-            if (token != CPP_IDENTIFIER) {
+            if (token != PpAtomIdentifier) {
                 parseContext.ppError(ppToken->loc, "bad argument", "#define", "");
 
                 return token;
@@ -197,18 +172,18 @@ int TPpContext::CPPdefine(TPpToken* ppToken)
     }
 
     // check for duplicate definition
-    symb = LookUpSymbol(atom);
+    symb = LookUpSymbol(defAtom);
     if (symb) {
         if (! symb->mac.undef) {
             // Already defined -- need to make sure they are identical:
             // "Two replacement lists are identical if and only if the preprocessing tokens in both have the same number,
             // ordering, spelling, and white-space separation, where all white-space separations are considered identical."
             if (symb->mac.argc != mac.argc)
-                parseContext.ppError(defineLoc, "Macro redefined; different number of arguments:", "#define", GetAtomString(atom));
+                parseContext.ppError(defineLoc, "Macro redefined; different number of arguments:", "#define", GetAtomString(defAtom));
             else {
                 for (int argc = 0; argc < mac.argc; argc++) {
                     if (symb->mac.args[argc] != mac.args[argc])
-                        parseContext.ppError(defineLoc, "Macro redefined; different argument names:", "#define", GetAtomString(atom));
+                        parseContext.ppError(defineLoc, "Macro redefined; different argument names:", "#define", GetAtomString(defAtom));
                 }
                 RewindTokenStream(symb->mac.body);
                 RewindTokenStream(mac.body);
@@ -220,14 +195,14 @@ int TPpContext::CPPdefine(TPpToken* ppToken)
                     oldToken = ReadToken(symb->mac.body, &oldPpToken);
                     newToken = ReadToken(mac.body, &newPpToken);
                     if (oldToken != newToken || oldPpToken != newPpToken) {
-                        parseContext.ppError(defineLoc, "Macro redefined; different substitutions:", "#define", GetAtomString(atom));
+                        parseContext.ppError(defineLoc, "Macro redefined; different substitutions:", "#define", GetAtomString(defAtom));
                         break; 
                     }
                 } while (newToken > 0);
             }
         }
     } else
-        symb = AddSymbol(atom);
+        symb = AddSymbol(defAtom);
 
     delete symb->mac.body;
     symb->mac = mac;
@@ -240,14 +215,13 @@ int TPpContext::CPPundef(TPpToken* ppToken)
 {
     int token = scanToken(ppToken);
     Symbol *symb;
-    if (token != CPP_IDENTIFIER) {
+    if (token != PpAtomIdentifier) {
         parseContext.ppError(ppToken->loc, "must be followed by macro name", "#undef", "");
 
         return token;
     }
 
-    const char* name = GetAtomString(ppToken->atom); // TODO preprocessor simplification: the token text should have been built into the ppToken during scanToken()
-    parseContext.reservedPpErrorCheck(ppToken->loc, name, "#undef");
+    parseContext.reservedPpErrorCheck(ppToken->loc, ppToken->name, "#undef");
 
     symb = LookUpSymbol(ppToken->atom);
     if (symb) {
@@ -283,15 +257,15 @@ int TPpContext::CPPelse(int matchelse, TPpToken* ppToken)
             continue;
         }
 
-        if ((token = scanToken(ppToken)) != CPP_IDENTIFIER)
+        if ((token = scanToken(ppToken)) != PpAtomIdentifier)
             continue;
 
         atom = ppToken->atom;
-        if (atom == ifAtom || atom == ifdefAtom || atom == ifndefAtom) {
+        if (atom == PpAtomIf || atom == PpAtomIfdef || atom == PpAtomIfndef) {
             depth++; 
             ifdepth++; 
             elsetracker++;
-        } else if (atom == endifAtom) {
+        } else if (atom == PpAtomEndif) {
             token = extraTokenCheck(atom, ppToken, scanToken(ppToken));
             elseSeen[elsetracker] = false;
             --elsetracker;
@@ -304,12 +278,12 @@ int TPpContext::CPPelse(int matchelse, TPpToken* ppToken)
             --depth;
             --ifdepth;
         } else if (matchelse && depth == 0) {
-            if (atom == elseAtom) {
+            if (atom == PpAtomElse) {
                 elseSeen[elsetracker] = true;
                 token = extraTokenCheck(atom, ppToken, scanToken(ppToken));
                 // found the #else we are looking for
                 break;
-            } else if (atom == elifAtom) {
+            } else if (atom == PpAtomElif) {
                 if (elseSeen[elsetracker])
                     parseContext.ppError(ppToken->loc, "#elif after #else", "#elif", "");
                 /* we decrement ifdepth here, because CPPif will increment
@@ -322,13 +296,13 @@ int TPpContext::CPPelse(int matchelse, TPpToken* ppToken)
 
                 return CPPif(ppToken);
             }
-        } else if (atom == elseAtom) {
+        } else if (atom == PpAtomElse) {
             if (elseSeen[elsetracker])
                 parseContext.ppError(ppToken->loc, "#else after #else", "#else", "");
             else
                 elseSeen[elsetracker] = true;
             token = extraTokenCheck(atom, ppToken, scanToken(ppToken));
-        } else if (atom == elifAtom) {
+        } else if (atom == PpAtomElif) {
             if (elseSeen[elsetracker])
                 parseContext.ppError(ppToken->loc, "#elif after #else", "#elif", "");
         }
@@ -344,15 +318,15 @@ int TPpContext::extraTokenCheck(int atom, TPpToken* ppToken, int token)
         static const char* message = "unexpected tokens following directive";
 
         const char* label;
-        if (atom == elseAtom)
+        if (atom == PpAtomElse)
             label = "#else";
-        else if (atom == elifAtom)
+        else if (atom == PpAtomElif)
             label = "#elif";
-        else if (atom == endifAtom)
+        else if (atom == PpAtomEndif)
             label = "#endif";
-        else if (atom == ifAtom)
+        else if (atom == PpAtomIf)
             label = "#if";
-        else if (atom == lineAtom)
+        else if (atom == PpAtomLine)
             label = "#line";
         else
             label = "";
@@ -405,19 +379,19 @@ namespace {
 struct TBinop {
     int token, precedence, (*op)(int, int);
 } binop[] = {
-    { CPP_OR_OP, LOGOR, op_logor },
-    { CPP_AND_OP, LOGAND, op_logand },
+    { PpAtomOr, LOGOR, op_logor },
+    { PpAtomAnd, LOGAND, op_logand },
     { '|', OR, op_or },
     { '^', XOR, op_xor },
     { '&', AND, op_and },
-    { CPP_EQ_OP, EQUAL, op_eq },
-    { CPP_NE_OP, EQUAL, op_ne },
+    { PpAtomEQ, EQUAL, op_eq },
+    { PpAtomNE, EQUAL, op_ne },
     { '>', RELATION, op_gt },
-    { CPP_GE_OP, RELATION, op_ge },
+    { PpAtomGE, RELATION, op_ge },
     { '<', RELATION, op_lt },
-    { CPP_LE_OP, RELATION, op_le },
-    { CPP_LEFT_OP, SHIFT, op_shl },
-    { CPP_RIGHT_OP, SHIFT, op_shr },
+    { PpAtomLE, RELATION, op_le },
+    { PpAtomLeft, SHIFT, op_shl },
+    { PpAtomRight, SHIFT, op_shr },
     { '+', ADD, op_add },
     { '-', ADD, op_sub },
     { '*', MUL, op_mul },
@@ -439,15 +413,15 @@ struct TUnop {
 int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, bool& err, TPpToken* ppToken)
 {
     TSourceLoc loc = ppToken->loc;  // because we sometimes read the newline before reporting the error
-    if (token == CPP_IDENTIFIER) {
-        if (ppToken->atom == definedAtom) {
+    if (token == PpAtomIdentifier) {
+        if (ppToken->atom == PpAtomDefined) {
             bool needclose = 0;
             token = scanToken(ppToken);
             if (token == '(') {
                 needclose = true;
                 token = scanToken(ppToken);
             }
-            if (token != CPP_IDENTIFIER) {
+            if (token != PpAtomIdentifier) {
                 parseContext.ppError(loc, "incorrect directive, expected identifier", "preprocessor evaluation", "");
                 err = true;
                 res = 0;
@@ -471,7 +445,7 @@ int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, boo
             token = evalToToken(token, shortCircuit, res, err, ppToken);
             return eval(token, precedence, shortCircuit, res, err, ppToken);
         }
-    } else if (token == CPP_INTCONSTANT) {
+    } else if (token == PpAtomConstInt) {
         res = ppToken->ival;
         token = scanToken(ppToken);
     } else if (token == '(') {
@@ -524,8 +498,8 @@ int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, boo
         // Setup short-circuiting, needed for ES, unless already in a short circuit.
         // (Once in a short-circuit, can't turn off again, until that whole subexpression is done.
         if (! shortCircuit) {
-            if ((token == CPP_OR_OP  && leftSide == 1) ||
-                (token == CPP_AND_OP && leftSide == 0))
+            if ((token == PpAtomOr  && leftSide == 1) ||
+                (token == PpAtomAnd && leftSide == 0))
                 shortCircuit = true;
         }
 
@@ -540,7 +514,7 @@ int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, boo
 // Expand macros, skipping empty expansions, to get to the first real token in those expansions.
 int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, TPpToken* ppToken)
 {
-    while (token == CPP_IDENTIFIER && ppToken->atom != definedAtom) {
+    while (token == PpAtomIdentifier && ppToken->atom != PpAtomDefined) {
         int macroReturn = MacroExpand(ppToken->atom, ppToken, true, false);
         if (macroReturn == 0) {
             parseContext.ppError(ppToken->loc, "can't evaluate expression", "preprocessor evaluation", "");
@@ -552,11 +526,10 @@ int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, T
         if (macroReturn == -1) {
             if (! shortCircuit && parseContext.profile == EEsProfile) {
                 const char* message = "undefined macro in expression not allowed in es profile";
-                const char* name = GetAtomString(ppToken->atom);
                 if (parseContext.relaxedErrors())
-                    parseContext.ppWarn(ppToken->loc, message, "preprocessor evaluation", name);
+                    parseContext.ppWarn(ppToken->loc, message, "preprocessor evaluation", ppToken->name);
                 else
-                    parseContext.ppError(ppToken->loc, message, "preprocessor evaluation", name);
+                    parseContext.ppError(ppToken->loc, message, "preprocessor evaluation", ppToken->name);
             }
         }
         token = scanToken(ppToken);
@@ -579,7 +552,7 @@ int TPpContext::CPPif(TPpToken* ppToken)
     int res = 0;
     bool err = false;
     token = eval(token, MIN_PRECEDENCE, false, res, err, ppToken);
-    token = extraTokenCheck(ifAtom, ppToken, token);
+    token = extraTokenCheck(PpAtomIf, ppToken, token);
     if (!res && !err)
         token = CPPelse(1, ppToken);
 
@@ -596,7 +569,7 @@ int TPpContext::CPPifdef(int defined, TPpToken* ppToken)
         return 0;
     }
     elsetracker++;
-    if (token != CPP_IDENTIFIER) {
+    if (token != PpAtomIdentifier) {
         if (defined)
             parseContext.ppError(ppToken->loc, "must be followed by macro name", "#ifdef", "");
         else 
@@ -656,7 +629,7 @@ int TPpContext::CPPline(TPpToken* ppToken)
     if (!fileErr && !lineErr) {
         parseContext.notifyLineDirective(directiveLoc, lineToken, hasFile, fileRes);
     }
-    token = extraTokenCheck(lineAtom, ppToken, token);
+    token = extraTokenCheck(PpAtomLine, ppToken, token);
 
     return token;
 }
@@ -669,11 +642,11 @@ int TPpContext::CPPerror(TPpToken* ppToken)
     TSourceLoc loc = ppToken->loc;
 
     while (token != '\n') {
-        if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
-            token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT) {
+        if (token == PpAtomConstInt || token == PpAtomConstUint ||
+            token == PpAtomConstFloat || token == PpAtomConstDouble) {
                 message.append(ppToken->name);
-        } else if (token == CPP_IDENTIFIER || token == CPP_STRCONSTANT) {
-            message.append(GetAtomString(ppToken->atom));
+        } else if (token == PpAtomIdentifier || token == PpAtomConstString) {
+            message.append(ppToken->name);
         } else {
             message.append(GetAtomString(token));
         }
@@ -691,23 +664,18 @@ int TPpContext::CPPerror(TPpToken* ppToken)
 int TPpContext::CPPpragma(TPpToken* ppToken)
 {
     char SrcStrName[2];
-    const char* SrcStr;
     TVector<TString> tokens;
 
     TSourceLoc loc = ppToken->loc;  // because we go to the next line before processing
     int token = scanToken(ppToken);
     while (token != '\n' && token != EOF) {
         switch (token) {
-        case CPP_IDENTIFIER:
-            SrcStr = GetAtomString(ppToken->atom);
-            tokens.push_back(SrcStr);
-            break;
-        case CPP_INTCONSTANT:
-        case CPP_UINTCONSTANT:
-        case CPP_FLOATCONSTANT:
-        case CPP_DOUBLECONSTANT:
-            SrcStr = ppToken->name;
-            tokens.push_back(SrcStr);
+        case PpAtomIdentifier:
+        case PpAtomConstInt:
+        case PpAtomConstUint:
+        case PpAtomConstFloat:
+        case PpAtomConstDouble:
+            tokens.push_back(ppToken->name);
             break;
         default:
             SrcStrName[0] = (char)token;
@@ -740,7 +708,7 @@ int TPpContext::CPPversion(TPpToken* ppToken)
         return token;
     }
 
-    if (token != CPP_INTCONSTANT)
+    if (token != PpAtomConstInt)
         parseContext.ppError(ppToken->loc, "must be followed by version number", "#version", "");
 
     ppToken->ival = atoi(ppToken->name);
@@ -752,11 +720,11 @@ int TPpContext::CPPversion(TPpToken* ppToken)
         parseContext.notifyVersion(line, versionNumber, nullptr);
         return token;
     } else {
-        if (ppToken->atom != coreAtom &&
-            ppToken->atom != compatibilityAtom &&
-            ppToken->atom != esAtom)
+        if (ppToken->atom != PpAtomCore &&
+            ppToken->atom != PpAtomCompatibility &&
+            ppToken->atom != PpAtomEs)
             parseContext.ppError(ppToken->loc, "bad profile name; use es, core, or compatibility", "#version", "");
-        parseContext.notifyVersion(line, versionNumber, GetAtomString(ppToken->atom));
+        parseContext.notifyVersion(line, versionNumber, ppToken->name);
         token = scanToken(ppToken);
 
         if (token == '\n')
@@ -780,10 +748,10 @@ int TPpContext::CPPextension(TPpToken* ppToken)
         return token;
     }
 
-    if (token != CPP_IDENTIFIER)
+    if (token != PpAtomIdentifier)
         parseContext.ppError(ppToken->loc, "extension name expected", "#extension", "");
 
-    strcpy(extensionName, GetAtomString(ppToken->atom));
+    strcpy(extensionName, ppToken->name);
 
     token = scanToken(ppToken);
     if (token != ':') {
@@ -792,12 +760,12 @@ int TPpContext::CPPextension(TPpToken* ppToken)
     }
 
     token = scanToken(ppToken);
-    if (token != CPP_IDENTIFIER) {
+    if (token != PpAtomIdentifier) {
         parseContext.ppError(ppToken->loc, "behavior for extension not specified", "#extension", "");
         return token;
     }
 
-    parseContext.updateExtensionBehavior(line, extensionName, GetAtomString(ppToken->atom));
+    parseContext.updateExtensionBehavior(line, extensionName, ppToken->name);
 
     token = scanToken(ppToken);
     if (token == '\n')
@@ -812,18 +780,21 @@ int TPpContext::readCPPline(TPpToken* ppToken)
 {
     int token = scanToken(ppToken);
 
-    if (token == CPP_IDENTIFIER) {
-        if (ppToken->atom == defineAtom) {
+    if (token == PpAtomIdentifier) {
+        switch (ppToken->atom) {
+        case PpAtomDefine:
             token = CPPdefine(ppToken);
-        } else if (ppToken->atom == elseAtom) {
+            break;
+        case PpAtomElse:
             if (elsetracker[elseSeen])
                 parseContext.ppError(ppToken->loc, "#else after #else", "#else", "");
             elsetracker[elseSeen] = true;
             if (! ifdepth)
                 parseContext.ppError(ppToken->loc, "mismatched statements", "#else", "");
-            token = extraTokenCheck(elseAtom, ppToken, scanToken(ppToken));
+            token = extraTokenCheck(PpAtomElse, ppToken, scanToken(ppToken));
             token = CPPelse(0, ppToken);
-        } else if (ppToken->atom == elifAtom) {
+            break;
+        case PpAtomElif:
             if (! ifdepth)
                 parseContext.ppError(ppToken->loc, "mismatched statements", "#elif", "");
             if (elseSeen[elsetracker])
@@ -833,34 +804,46 @@ int TPpContext::readCPPline(TPpToken* ppToken)
             while (token != '\n')
                 token = scanToken(ppToken);
             token = CPPelse(0, ppToken);
-        } else if (ppToken->atom == endifAtom) {
+            break;
+        case PpAtomEndif:
             elseSeen[elsetracker] = false;
             --elsetracker;
             if (! ifdepth)
                 parseContext.ppError(ppToken->loc, "mismatched statements", "#endif", "");
             else
                 --ifdepth;
-            token = extraTokenCheck(endifAtom, ppToken, scanToken(ppToken));
-        } else if (ppToken->atom == ifAtom) {
-            token = CPPif (ppToken);
-        } else if (ppToken->atom == ifdefAtom) {
+            token = extraTokenCheck(PpAtomEndif, ppToken, scanToken(ppToken));
+            break;
+        case PpAtomIf:
+            token = CPPif(ppToken);
+            break;
+        case PpAtomIfdef:
             token = CPPifdef(1, ppToken);
-        } else if (ppToken->atom == ifndefAtom) {
+            break;
+        case PpAtomIfndef:
             token = CPPifdef(0, ppToken);
-        } else if (ppToken->atom == lineAtom) {
+            break;
+        case PpAtomLine:
             token = CPPline(ppToken);
-        } else if (ppToken->atom == pragmaAtom) {
+            break;
+        case PpAtomPragma:
             token = CPPpragma(ppToken);
-        } else if (ppToken->atom == undefAtom) {
+            break;
+        case PpAtomUndef:
             token = CPPundef(ppToken);
-        } else if (ppToken->atom == errorAtom) {
+            break;
+        case PpAtomError:
             token = CPPerror(ppToken);
-        } else if (ppToken->atom == versionAtom) {
+            break;
+        case PpAtomVersion:
             token = CPPversion(ppToken);
-        } else if (ppToken->atom == extensionAtom) {
+            break;
+        case PpAtomExtension:
             token = CPPextension(ppToken);
-        } else {
-            parseContext.ppError(ppToken->loc, "invalid directive:", "#", GetAtomString(ppToken->atom));
+            break;
+        default:
+            parseContext.ppError(ppToken->loc, "invalid directive:", "#", ppToken->name);
+            break;
         }
     } else if (token != '\n' && token != EOF)
         parseContext.ppError(ppToken->loc, "invalid directive", "#", "");
@@ -878,7 +861,7 @@ TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* p
     RewindTokenStream(a);
     do {
         token = ReadToken(a, ppToken);
-        if (token == CPP_IDENTIFIER && LookUpSymbol(ppToken->atom))
+        if (token == PpAtomIdentifier && LookUpSymbol(ppToken->atom))
             break;
     } while (token != tInput::endOfInput);
 
@@ -889,7 +872,7 @@ TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* p
     pushInput(new tMarkerInput(this));
     pushTokenStreamInput(a);
     while ((token = scanToken(ppToken)) != tMarkerInput::marker) {
-        if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0)
+        if (token == PpAtomIdentifier && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0)
             continue;
         RecordToken(n, token, ppToken);
     }
@@ -910,7 +893,7 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
     } while (token == ' ');  // handle white space in macro
 
     // TODO: preprocessor:  properly handle whitespace (or lack of it) between tokens when expanding
-    if (token == CPP_IDENTIFIER) {
+    if (token == PpAtomIdentifier) {
         int i;
         for (i = mac->argc - 1; i >= 0; i--)
             if (mac->args[i] == ppToken->atom) 
@@ -939,7 +922,7 @@ int TPpContext::tZeroInput::scan(TPpToken* ppToken)
     ppToken->space = false;
     done = true;
 
-    return CPP_INTCONSTANT;
+    return PpAtomConstInt;
 }
 
 //
@@ -952,35 +935,34 @@ int TPpContext::tZeroInput::scan(TPpToken* ppToken)
 //
 int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool newLineOkay)
 {
-    Symbol *sym = LookUpSymbol(atom);
-    int token;
-    int depth = 0;
-
     ppToken->space = false;
-    if (atom == __LINE__Atom) {
+    switch (atom) {
+    case PpAtomLineMacro:
         ppToken->ival = parseContext.getCurrentLoc().line;
         sprintf(ppToken->name, "%d", ppToken->ival);
-        UngetToken(CPP_INTCONSTANT, ppToken);
-
+        UngetToken(PpAtomConstInt, ppToken);
         return 1;
-    }
 
-    if (atom == __FILE__Atom) {
+    case PpAtomFileMacro:
         ppToken->ival = parseContext.getCurrentLoc().string;
         sprintf(ppToken->name, "%d", ppToken->ival);
-        UngetToken(CPP_INTCONSTANT, ppToken);
-
+        UngetToken(PpAtomConstInt, ppToken);
         return 1;
-    }
 
-    if (atom == __VERSION__Atom) {
+    case PpAtomVersionMacro:
         ppToken->ival = parseContext.version;
         sprintf(ppToken->name, "%d", ppToken->ival);
-        UngetToken(CPP_INTCONSTANT, ppToken);
-
+        UngetToken(PpAtomConstInt, ppToken);
         return 1;
+
+    default:
+        break;
     }
 
+    Symbol *sym = LookUpSymbol(atom);
+    int token;
+    int depth = 0;
+
     // no recursive expansions
     if (sym && sym->mac.busy)
         return 0;

diff --git a/glslang/MachineIndependent/preprocessor/PpAtom.cpp b/glslang/MachineIndependent/preprocessor/PpAtom.cpp
index d058d5d..58f6515 100644 (file)
--- a/glslang/MachineIndependent/preprocessor/PpAtom.cpp
+++ b/glslang/MachineIndependent/preprocessor/PpAtom.cpp
@@ -98,27 +98,28 @@ const struct {
     int val;
     const char* str;
 } tokens[] = {
-    { CPP_AND_OP,         "&&" },
-    { CPP_AND_ASSIGN,     "&=" },
-    { CPP_SUB_ASSIGN,     "-=" },
-    { CPP_MOD_ASSIGN,     "%=" },
-    { CPP_ADD_ASSIGN,     "+=" },
-    { CPP_DIV_ASSIGN,     "/=" },
-    { CPP_MUL_ASSIGN,     "*=" },
-    { CPP_EQ_OP,          "==" },
-    { CPP_XOR_OP,         "^^" }, 
-    { CPP_XOR_ASSIGN,     "^=" }, 
-    { CPP_GE_OP,          ">=" },
-    { CPP_RIGHT_OP,       ">>" },
-    { CPP_RIGHT_ASSIGN,   ">>="}, 
-    { CPP_LE_OP,          "<=" },
-    { CPP_LEFT_OP,        "<<" },
-    { CPP_LEFT_ASSIGN,    "<<="},
-    { CPP_DEC_OP,         "--" },
-    { CPP_NE_OP,          "!=" },
-    { CPP_OR_OP,          "||" },
-    { CPP_OR_ASSIGN,      "|=" }, 
-    { CPP_INC_OP,         "++" },
+    { PpAtomDefine,         "define" },
+    { PpAtomDefined,        "defined" },
+    { PpAtomUndef,          "undef" },
+    { PpAtomIf,             "if" },
+    { PpAtomElif,           "elif" },
+    { PpAtomElse,           "else" },
+    { PpAtomEndif,          "endif" },
+    { PpAtomIfdef,          "ifdef" },
+    { PpAtomIfndef,         "ifndef" },
+    { PpAtomLine,           "line" },
+    { PpAtomPragma,         "pragma" },
+    { PpAtomError,          "error" },
+
+    { PpAtomVersion,        "version" },
+    { PpAtomCore,           "core" },
+    { PpAtomCompatibility,  "compatibility" },
+    { PpAtomEs,             "es" },
+    { PpAtomExtension,      "extension" },
+
+    { PpAtomLineMacro,       "__LINE__" },
+    { PpAtomFileMacro,       "__FILE__" },
+    { PpAtomVersionMacro,    "__VERSION__" },
 };
 
 } // end anonymous namespace
@@ -131,9 +132,10 @@ namespace glslang {
 int TPpContext::LookUpAddString(const char* s)
 {
     auto it = atomMap.find(s);
-    if (it == atomMap.end())
-        return AddAtomFixed(s, nextAtom++);
-    else
+    if (it == atomMap.end()) {
+        AddAtomFixed(s, nextAtom);
+        return nextAtom++;
+    } else
         return it->second;
 }
 
@@ -159,14 +161,12 @@ const char* TPpContext::GetAtomString(int atom)
 //
 // Add forced mapping of string to atom.
 //
-int TPpContext::AddAtomFixed(const char* s, int atom)
+void TPpContext::AddAtomFixed(const char* s, int atom)
 {
     auto it = atomMap.insert(std::pair<TString, int>(s, atom)).first;
     if (stringMap.size() < (size_t)atom + 1)
         stringMap.resize(atom + 100, 0);
     stringMap[atom] = &it->first;
-
-    return atom;
 }
 
 //
@@ -189,7 +189,7 @@ void TPpContext::InitAtomTable()
     for (int ii = 0; ii < sizeof(tokens)/sizeof(tokens[0]); ii++)
         AddAtomFixed(tokens[ii].str, tokens[ii].val);
 
-    nextAtom = CPP_FIRST_USER_TOKEN_SY;
+    nextAtom = PpAtomLast;
 }
 
 } // end namespace glslang

diff --git a/glslang/MachineIndependent/preprocessor/PpContext.cpp b/glslang/MachineIndependent/preprocessor/PpContext.cpp
index 41d7f4e..828764b 100644 (file)
--- a/glslang/MachineIndependent/preprocessor/PpContext.cpp
+++ b/glslang/MachineIndependent/preprocessor/PpContext.cpp
@@ -93,48 +93,6 @@ TPpContext::TPpContext(TParseContext& pc) :
     for (elsetracker = 0; elsetracker < maxIfNesting; elsetracker++)
         elseSeen[elsetracker] = false;
     elsetracker = 0;
-    
-    // The following identifies all legal characters in GLSL:
-
-    //for (int c = 0; c < 256; ++c)
-    //    languageCharacters[c] = false;
-    //for (int c = 'a'; c <= 'z'; ++c)
-    //    languageCharacters[c] = true;
-    //for (int c = 'A'; c <= 'Z'; ++c)
-    //    languageCharacters[c] = true;
-    //languageCharacters['_'] = true;
-    //for (int c = '0'; c <= '9'; ++c)
-    //    languageCharacters[c] = true;
-    //languageCharacters['.'] = true;
-    //languageCharacters['+'] = true;
-    //languageCharacters['-'] = true;
-    //languageCharacters['/'] = true;
-    //languageCharacters['*'] = true;
-    //languageCharacters['%'] = true;
-    //languageCharacters['<'] = true;
-    //languageCharacters['>'] = true;
-    //languageCharacters['['] = true;
-    //languageCharacters[']'] = true;
-    //languageCharacters['('] = true;
-    //languageCharacters[')'] = true;
-    //languageCharacters['{'] = true;
-    //languageCharacters['}'] = true;
-    //languageCharacters['^'] = true;
-    //languageCharacters['|'] = true;
-    //languageCharacters['&'] = true;
-    //languageCharacters['~'] = true;
-    //languageCharacters['='] = true;
-    //languageCharacters['!'] = true;
-    //languageCharacters[':'] = true;
-    //languageCharacters[';'] = true;
-    //languageCharacters[','] = true;
-    //languageCharacters['?'] = true;
-    //languageCharacters['#'] = true;
-
-    //// white space
-    //languageCharacters[' '] = true;
-    //for (int c = 9; c <= 13; ++c)
-    //    languageCharacters[c] = true;
 }
 
 TPpContext::~TPpContext()

diff --git a/glslang/MachineIndependent/preprocessor/PpContext.h b/glslang/MachineIndependent/preprocessor/PpContext.h
index dadf706..269312b 100644 (file)
--- a/glslang/MachineIndependent/preprocessor/PpContext.h
+++ b/glslang/MachineIndependent/preprocessor/PpContext.h
@@ -287,31 +287,6 @@ protected:
     //
     // from Pp.cpp
     //
-    int bindAtom;
-    int constAtom;
-    int defaultAtom;
-    int defineAtom;
-    int definedAtom;
-    int elseAtom;
-    int elifAtom;
-    int endifAtom;
-    int ifAtom;
-    int ifdefAtom;
-    int ifndefAtom;
-    int includeAtom;
-    int lineAtom;
-    int pragmaAtom;
-    int texunitAtom;
-    int undefAtom;
-    int errorAtom;
-    int __LINE__Atom;
-    int __FILE__Atom;
-    int __VERSION__Atom;
-    int versionAtom;
-    int coreAtom;
-    int compatibilityAtom;
-    int esAtom;
-    int extensionAtom;
     TSourceLoc ifloc; /* outermost #if */
 
     int InitCPP();
@@ -467,7 +442,7 @@ protected:
     TStringMap stringMap;
     int nextAtom;
     void InitAtomTable();
-    int AddAtomFixed(const char* s, int atom);
+    void AddAtomFixed(const char* s, int atom);
     int LookUpAddString(const char* s);
     const char* GetAtomString(int atom);
 

diff --git a/glslang/MachineIndependent/preprocessor/PpScanner.cpp b/glslang/MachineIndependent/preprocessor/PpScanner.cpp
index 105e327..f7283d0 100644 (file)
--- a/glslang/MachineIndependent/preprocessor/PpScanner.cpp
+++ b/glslang/MachineIndependent/preprocessor/PpScanner.cpp
@@ -223,9 +223,9 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
     }
 
     if (isDouble)
-        return CPP_DOUBLECONSTANT;
+        return PpAtomConstDouble;
     else
-        return CPP_FLOATCONSTANT;
+        return PpAtomConstFloat;
 }
 
 //
@@ -233,7 +233,7 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
 //
 int TPpContext::tStringInput::scan(TPpToken* ppToken)
 {
-    char tokenText[TPpToken::maxTokenLength + 1];
+    char* tokenText = ppToken->name;
     int AlreadyComplained = 0;
     int len = 0;
     int ch = 0;
@@ -253,7 +253,8 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
         len = 0;
         switch (ch) {
         default:
-            return ch; // Single character token, including '#' and '\' (escaped newlines are handled at a lower level, so this is just a '\' token)
+            // Single character token, including '#' and '\' (escaped newlines are handled at a lower level, so this is just a '\' token)
+            return ch;
 
         case EOF:
             return endOfInput;
@@ -273,7 +274,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
             do {
                 if (len < TPpToken::maxTokenLength) {
                     tokenText[len++] = (char)ch;
-                    ch = pp->getChar();                                        
+                    ch = pp->getChar();
                 } else {
                     if (! AlreadyComplained) {
                         pp->parseContext.ppError(ppToken->loc, "name too long", "", "");
@@ -293,8 +294,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
             tokenText[len] = '\0';
             pp->ungetChar();
             ppToken->atom = pp->LookUpAddString(tokenText);
-
-            return CPP_IDENTIFIER;
+            return PpAtomIdentifier;
         case '0':
             ppToken->name[len++] = (char)ch;
             ch = pp->getChar();
@@ -345,9 +345,9 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
                 ppToken->ival = (int)ival;
 
                 if (isUnsigned)
-                    return CPP_UINTCONSTANT;
+                    return PpAtomConstUint;
                 else
-                    return CPP_INTCONSTANT;
+                    return PpAtomConstInt;
             } else {
                 // could be octal integer or floating point, speculative pursue octal until it must be floating point
 
@@ -406,9 +406,9 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
                 ppToken->ival = (int)ival;
 
                 if (isUnsigned)
-                    return CPP_UINTCONSTANT;
+                    return PpAtomConstUint;
                 else
-                    return CPP_INTCONSTANT;
+                    return PpAtomConstInt;
             }
             break;
         case '1': case '2': case '3': case '4':
@@ -453,17 +453,17 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
                 ppToken->ival = (int)ival;
 
                 if (uint)
-                    return CPP_UINTCONSTANT;
+                    return PpAtomConstUint;
                 else
-                    return CPP_INTCONSTANT;
+                    return PpAtomConstInt;
             }
             break;
         case '-':
             ch = pp->getChar();
             if (ch == '-') {
-                return CPP_DEC_OP;
+                return PpAtomDecrement;
             } else if (ch == '=') {
-                return CPP_SUB_ASSIGN;
+                return PpAtomSub;
             } else {
                 pp->ungetChar();
                 return '-';
@@ -471,9 +471,9 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
         case '+':
             ch = pp->getChar();
             if (ch == '+') {
-                return CPP_INC_OP;
+                return PpAtomIncrement;
             } else if (ch == '=') {
-                return CPP_ADD_ASSIGN;
+                return PpAtomAdd;
             } else {
                 pp->ungetChar();
                 return '+';
@@ -481,7 +481,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
         case '*':
             ch = pp->getChar();
             if (ch == '=') {
-                return CPP_MUL_ASSIGN;
+                return PpAtomMul;
             } else {
                 pp->ungetChar();
                 return '*';
@@ -489,28 +489,18 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
         case '%':
             ch = pp->getChar();
             if (ch == '=') {
-                return CPP_MOD_ASSIGN;
-            } else if (ch == '>'){
-                return CPP_RIGHT_BRACE;
+                return PpAtomMod;
             } else {
                 pp->ungetChar();
                 return '%';
             }
-        case ':':
-            ch = pp->getChar();
-            if (ch == '>') {
-                return CPP_RIGHT_BRACKET;
-            } else {
-                pp->ungetChar();
-                return ':';
-            }
         case '^':
             ch = pp->getChar();
             if (ch == '^') {
-                return CPP_XOR_OP;
+                return PpAtomXor;
             } else {
                 if (ch == '=')
-                    return CPP_XOR_ASSIGN;
+                    return PpAtomXorAssign;
                 else{
                     pp->ungetChar();
                     return '^';
@@ -520,7 +510,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
         case '=':
             ch = pp->getChar();
             if (ch == '=') {
-                return CPP_EQ_OP;
+                return PpAtomEQ;
             } else {
                 pp->ungetChar();
                 return '=';
@@ -528,7 +518,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
         case '!':
             ch = pp->getChar();
             if (ch == '=') {
-                return CPP_NE_OP;
+                return PpAtomNE;
             } else {
                 pp->ungetChar();
                 return '!';
@@ -536,68 +526,54 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
         case '|':
             ch = pp->getChar();
             if (ch == '|') {
-                return CPP_OR_OP;
+                return PpAtomOr;
+            } else if (ch == '=') {
+                return PpAtomOrAssign;
             } else {
-                if (ch == '=')
-                    return CPP_OR_ASSIGN;
-                else{
-                    pp->ungetChar();
-                    return '|';
-                }
+                pp->ungetChar();
+                return '|';
             }
         case '&':
             ch = pp->getChar();
             if (ch == '&') {
-                return CPP_AND_OP;
+                return PpAtomAnd;
+            } else if (ch == '=') {
+                return PpAtomAndAssign;
             } else {
-                if (ch == '=')
-                    return CPP_AND_ASSIGN;
-                else{
-                    pp->ungetChar();
-                    return '&';
-                }
+                pp->ungetChar();
+                return '&';
             }
         case '<':
             ch = pp->getChar();
             if (ch == '<') {
                 ch = pp->getChar();
                 if (ch == '=')
-                    return CPP_LEFT_ASSIGN;
-                else{
+                    return PpAtomLeftAssign;
+                else {
                     pp->ungetChar();
-                    return CPP_LEFT_OP;
+                    return PpAtomLeft;
                 }
+            } else if (ch == '=') {
+                return PpAtomLE;
             } else {
-                if (ch == '=') {
-                    return CPP_LE_OP;
-                } else {
-                    if (ch == '%')
-                        return CPP_LEFT_BRACE;
-                    else if (ch == ':')
-                        return CPP_LEFT_BRACKET;
-                    else{
-                        pp->ungetChar();
-                        return '<';
-                    }
-                }
+                pp->ungetChar();
+                return '<';
             }
         case '>':
             ch = pp->getChar();
             if (ch == '>') {
                 ch = pp->getChar();
                 if (ch == '=')
-                    return CPP_RIGHT_ASSIGN;
-                else{
+                    return PpAtomRightAssign;
+                else {
                     pp->ungetChar();
-                    return CPP_RIGHT_OP;
+                    return PpAtomRight;
                 }
+            } else if (ch == '=') {
+                return PpAtomGE;
             } else {
-                if (ch == '=') {
-                    return CPP_GE_OP;
-                } else {
-                    pp->ungetChar();
-                    return '>';
-                }
+                pp->ungetChar();
+                return '>';
             }
         case '.':
             ch = pp->getChar();
@@ -642,7 +618,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
                 // loop again to get the next token...
                 break;
             } else if (ch == '=') {
-                return CPP_DIV_ASSIGN;
+                return PpAtomDiv;
             } else {
                 pp->ungetChar();
                 return '/';
@@ -659,13 +635,11 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
                     break;
             };
             tokenText[len] = '\0';
-            if (ch == '"') {
-                ppToken->atom = pp->LookUpAddString(tokenText);
-                return CPP_STRCONSTANT;
-            } else {
+            if (ch != '"') {
+                pp->ungetChar();
                 pp->parseContext.ppError(ppToken->loc, "end of line in string", "string", "");
-                return CPP_ERROR_SY;
             }
+            return PpAtomConstString;
         }
 
         ch = pp->getChar();
@@ -684,7 +658,6 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
     int token = '\n';
 
     for(;;) {
-        const char* tokenString = nullptr;
         token = scanToken(ppToken);
         ppToken->token = token;
         if (token == EOF) {
@@ -710,22 +683,28 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
             continue;
 
         // expand macros
-        if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false, true) != 0)
+        if (token == PpAtomIdentifier && MacroExpand(ppToken->atom, ppToken, false, true) != 0)
             continue;
 
-        if (token == CPP_IDENTIFIER)
-            tokenString = GetAtomString(ppToken->atom);
-        else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
-                 token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT)
+        const char* tokenString = nullptr;
+        switch (token) {
+        case PpAtomIdentifier:
+        case PpAtomConstInt:
+        case PpAtomConstUint:
+        case PpAtomConstFloat:
+        case PpAtomConstDouble:
             tokenString = ppToken->name;
-        else if (token == CPP_STRCONSTANT) {
+            break;
+        case PpAtomConstString:
             parseContext.ppError(ppToken->loc, "string literals not supported", "\"\"", "");
-            tokenString = nullptr;
-        } else if (token == '\'') {
+            break;
+        case '\'':
             parseContext.ppError(ppToken->loc, "character literals not supported", "\'", "");
-            tokenString = nullptr;
-        } else
+            break;
+        default:
             tokenString = GetAtomString(token);
+            break;
+        }
 
         if (tokenString) {
             if (tokenString[0] != 0)

diff --git a/glslang/MachineIndependent/preprocessor/PpTokens.cpp b/glslang/MachineIndependent/preprocessor/PpTokens.cpp
index 794cb24..8476024 100644 (file)
--- a/glslang/MachineIndependent/preprocessor/PpTokens.cpp
+++ b/glslang/MachineIndependent/preprocessor/PpTokens.cpp
@@ -126,25 +126,25 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken)
     const char* s;
     char* str = NULL;
 
-    if (token > 256)
+    if (token > PpAtomMaxSingle)
         lAddByte(pTok, (unsigned char)((token & 0x7f) + 0x80));
     else
         lAddByte(pTok, (unsigned char)(token & 0x7f));
 
     switch (token) {
-    case CPP_IDENTIFIER:
-    case CPP_STRCONSTANT:
-        s = GetAtomString(ppToken->atom);
+    case PpAtomIdentifier:
+    case PpAtomConstString:
+        s = ppToken->name;
         while (*s)
             lAddByte(pTok, (unsigned char) *s++);
         lAddByte(pTok, 0);
         break;
-    case CPP_INTCONSTANT:
-    case CPP_UINTCONSTANT:
-    case CPP_FLOATCONSTANT:
-    case CPP_DOUBLECONSTANT:
+    case PpAtomConstInt:
+    case PpAtomConstUint:
+    case PpAtomConstFloat:
+    case PpAtomConstDouble:
         str = ppToken->name;
-        while (*str){
+        while (*str) {
             lAddByte(pTok, (unsigned char) *str);
             str++;
         }
@@ -168,7 +168,7 @@ void TPpContext::RewindTokenStream(TokenStream *pTok)
 */
 int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
 {
-    char tokenText[TPpToken::maxTokenLength + 1];
+    char* tokenText = ppToken->name;
     int ltoken, len;
     int ch;
 
@@ -182,17 +182,17 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
             parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)");
             parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)");
             parseContext.error(ppToken->loc, "token pasting not implemented (internal error)", "##", "");
-            //return CPP_TOKEN_PASTE;
+            //return PpAtomPaste;
             return ReadToken(pTok, ppToken);
         } else
             lUnreadByte(pTok);
         break;
-    case CPP_STRCONSTANT:
-    case CPP_IDENTIFIER:
-    case CPP_FLOATCONSTANT:
-    case CPP_DOUBLECONSTANT:
-    case CPP_INTCONSTANT:
-    case CPP_UINTCONSTANT:
+    case PpAtomConstString:
+    case PpAtomIdentifier:
+    case PpAtomConstFloat:
+    case PpAtomConstDouble:
+    case PpAtomConstInt:
+    case PpAtomConstUint:
         len = 0;
         ch = lReadByte(pTok);
         while (ch != 0) {
@@ -208,17 +208,18 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
         tokenText[len] = 0;
 
         switch (ltoken) {
-        case CPP_IDENTIFIER:
-        case CPP_STRCONSTANT:
+        case PpAtomIdentifier:
             ppToken->atom = LookUpAddString(tokenText);
             break;
-        case CPP_FLOATCONSTANT:
-        case CPP_DOUBLECONSTANT:
+        case PpAtomConstString:
+            break;
+        case PpAtomConstFloat:
+        case PpAtomConstDouble:
             strcpy(ppToken->name, tokenText);
             ppToken->dval = atof(ppToken->name);
             break;
-        case CPP_INTCONSTANT:
-        case CPP_UINTCONSTANT:
+        case PpAtomConstInt:
+        case PpAtomConstUint:
             strcpy(ppToken->name, tokenText);
             if (len > 0 && tokenText[0] == '0') {
                 if (len > 1 && (tokenText[1] == 'x' || tokenText[1] == 'X'))

diff --git a/glslang/MachineIndependent/preprocessor/PpTokens.h b/glslang/MachineIndependent/preprocessor/PpTokens.h
index 022fc33..0d116f9 100644 (file)
--- a/glslang/MachineIndependent/preprocessor/PpTokens.h
+++ b/glslang/MachineIndependent/preprocessor/PpTokens.h
@@ -78,40 +78,89 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef PARSER_H
 #define PARSER_H
 
-#define CPP_AND_OP         257
-#define CPP_SUB_ASSIGN     259
-#define CPP_MOD_ASSIGN     260
-#define CPP_ADD_ASSIGN     261
-#define CPP_DIV_ASSIGN     262
-#define CPP_MUL_ASSIGN     263
-#define CPP_EQ_OP          264
-#define CPP_XOR_OP         265 
-#define CPP_ERROR_SY       266
-#define CPP_FLOATCONSTANT  267
-#define CPP_GE_OP          268
-#define CPP_RIGHT_OP       269
-#define CPP_IDENTIFIER     270
-#define CPP_INTCONSTANT    271
-#define CPP_LE_OP          272
-#define CPP_LEFT_OP        273
-#define CPP_DEC_OP         274
-#define CPP_NE_OP          275
-#define CPP_OR_OP          276
-#define CPP_INC_OP         277
-#define CPP_STRCONSTANT    278
-#define CPP_RIGHT_ASSIGN   280
-#define CPP_LEFT_ASSIGN    281
-#define CPP_AND_ASSIGN     282
-#define CPP_OR_ASSIGN      283
-#define CPP_XOR_ASSIGN     284
-#define CPP_LEFT_BRACKET   285
-#define CPP_RIGHT_BRACKET  286
-#define CPP_LEFT_BRACE     287
-#define CPP_RIGHT_BRACE    288
-#define CPP_UINTCONSTANT   289
-#define CPP_DOUBLECONSTANT 290
-#define CPP_TOKEN_PASTE    291
-
-#define CPP_FIRST_USER_TOKEN_SY 292
+namespace glslang {
+
+// Multi-character tokens
+enum EFixedAtoms {
+    PpAtomMaxSingle = 256, // single character tokens get their own char value as their token, skip them
+
+    // Operators
+
+    PpAtomAdd,
+    PpAtomSub,
+    PpAtomMul,
+    PpAtomDiv,
+    PpAtomMod,
+
+    PpAtomRight,
+    PpAtomLeft,
+
+    PpAtomRightAssign,
+    PpAtomLeftAssign,
+    PpAtomAndAssign,
+    PpAtomOrAssign,
+    PpAtomXorAssign,
+
+    PpAtomAnd,
+    PpAtomOr,
+    PpAtomXor,
+
+    PpAtomEQ,
+    PpAtomNE,
+    PpAtomGE,
+    PpAtomLE,
+
+    PpAtomDecrement,
+    PpAtomIncrement,
+
+    PpAtomPaste,
+
+    // Constants
+
+    PpAtomConstInt,
+    PpAtomConstUint,
+    PpAtomConstFloat,
+    PpAtomConstDouble,
+    PpAtomConstString,
+
+    // Indentifiers 
+    PpAtomIdentifier,
+
+    // preprocessor "keywords"
+
+    PpAtomDefine,
+    PpAtomDefined,
+    PpAtomUndef,
+
+    PpAtomIf,
+    PpAtomIfdef,
+    PpAtomIfndef,
+    PpAtomElse,
+    PpAtomElif,
+    PpAtomEndif,
+
+    PpAtomLine,
+    PpAtomPragma,
+    PpAtomError,
+
+    // #version ...
+    PpAtomVersion,
+    PpAtomCore,
+    PpAtomCompatibility,
+    PpAtomEs,
+
+    // #extension
+    PpAtomExtension,
+
+    // __LINE__, __FILE__, __VERSION__
+
+    PpAtomLineMacro,
+    PpAtomFileMacro,
+    PpAtomVersionMacro,
+
+    PpAtomLast,
+};
+
+} // end namespace glslang
 
 #endif /* not PARSER_H */