From b49bb2ca5c1fd577510085452375dbc89ef51138 Mon Sep 17 00:00:00 2001 From: John Kessenich Date: Fri, 10 Feb 2017 12:56:05 -0700 Subject: [PATCH] PP, nonfunctional: Remove crufty bit-twiddling of tokens. --- Test/baseResults/badChars.frag.out | 2 +- glslang/Include/revision.h | 2 +- glslang/MachineIndependent/preprocessor/Pp.cpp | 12 ++-- .../MachineIndependent/preprocessor/PpContext.h | 6 +- .../MachineIndependent/preprocessor/PpScanner.cpp | 2 + .../MachineIndependent/preprocessor/PpTokens.cpp | 73 ++++++++++------------ glslang/MachineIndependent/preprocessor/PpTokens.h | 6 +- 7 files changed, 52 insertions(+), 51 deletions(-) diff --git a/Test/baseResults/badChars.frag.out b/Test/baseResults/badChars.frag.out index a29b22f..e9ebff7 100644 --- a/Test/baseResults/badChars.frag.out +++ b/Test/baseResults/badChars.frag.out @@ -4,7 +4,7 @@ ERROR: 0:1: '#if' : unexpected tokens following directive ERROR: 0:3: '#error' : A B ERROR: 0:4: 'preprocessor evaluation' : bad expression ERROR: 0:4: '#if' : unexpected tokens following directive -ERROR: 0:6: 'ÿ' : unexpected token +ERROR: 0:6: '€' : unexpected token ERROR: 0:7: '' : syntax error ERROR: 7 compilation errors. No code generated. diff --git a/glslang/Include/revision.h b/glslang/Include/revision.h index a45d07f..18a0625 100644 --- a/glslang/Include/revision.h +++ b/glslang/Include/revision.h @@ -2,5 +2,5 @@ // For the version, it uses the latest git tag followed by the number of commits. // For the date, it uses the current date (when then script is run). -#define GLSLANG_REVISION "Overload400-PrecQual.1820" +#define GLSLANG_REVISION "Overload400-PrecQual.1825" #define GLSLANG_DATE "10-Feb-2017" diff --git a/glslang/MachineIndependent/preprocessor/Pp.cpp b/glslang/MachineIndependent/preprocessor/Pp.cpp index 6dd02ca..e6477f8 100644 --- a/glslang/MachineIndependent/preprocessor/Pp.cpp +++ b/glslang/MachineIndependent/preprocessor/Pp.cpp @@ -1076,16 +1076,16 @@ bool TPpContext::tMacroInput::peekMacPasting() size_t savePos = mac->body.current; // skip white-space - int ltoken; + int subtoken; do { - ltoken = pp->lReadByte(mac->body); - } while (ltoken == ' '); + subtoken = pp->getSubtoken(mac->body); + } while (subtoken == ' '); // check for ## bool pasting = false; - if (ltoken == '#') { - ltoken = pp->lReadByte(mac->body); - if (ltoken == '#') + if (subtoken == '#') { + subtoken = pp->getSubtoken(mac->body); + if (subtoken == '#') pasting = true; } diff --git a/glslang/MachineIndependent/preprocessor/PpContext.h b/glslang/MachineIndependent/preprocessor/PpContext.h index 15e54b7..d7702f8 100644 --- a/glslang/MachineIndependent/preprocessor/PpContext.h +++ b/glslang/MachineIndependent/preprocessor/PpContext.h @@ -375,9 +375,9 @@ protected: // // From PpTokens.cpp // - void lAddByte(TokenStream&, unsigned char fVal); - int lReadByte(TokenStream&); - void lUnreadByte(TokenStream&); + void putSubtoken(TokenStream&, int fVal); + int getSubtoken(TokenStream&); + void ungetSubtoken(TokenStream&); void RecordToken(TokenStream&, int token, TPpToken* ppToken); void RewindTokenStream(TokenStream&); int ReadToken(TokenStream&, TPpToken*); diff --git a/glslang/MachineIndependent/preprocessor/PpScanner.cpp b/glslang/MachineIndependent/preprocessor/PpScanner.cpp index 83e6c89..a304424 100644 --- a/glslang/MachineIndependent/preprocessor/PpScanner.cpp +++ b/glslang/MachineIndependent/preprocessor/PpScanner.cpp @@ -232,6 +232,8 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken) switch (ch) { default: // Single character token, including EndOfInput, '#' and '\' (escaped newlines are handled at a lower level, so this is just a '\' token) + if (ch > PpAtomMaxSingle) + ch = PpAtomBadToken; return ch; case 'A': case 'B': case 'C': case 'D': case 'E': diff --git a/glslang/MachineIndependent/preprocessor/PpTokens.cpp b/glslang/MachineIndependent/preprocessor/PpTokens.cpp index a8fc9ac..660d73c 100644 --- a/glslang/MachineIndependent/preprocessor/PpTokens.cpp +++ b/glslang/MachineIndependent/preprocessor/PpTokens.cpp @@ -95,26 +95,27 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace glslang { -void TPpContext::lAddByte(TokenStream& fTok, unsigned char fVal) +// push onto back of stream +void TPpContext::putSubtoken(TokenStream& stream, int subtoken) { - fTok.data.push_back(fVal); + assert((subtoken & ~0xff) == 0); + stream.data.push_back(static_cast(subtoken)); } -/* -* Get the next byte from a stream. -*/ -int TPpContext::lReadByte(TokenStream& pTok) +// get the next token in stream +int TPpContext::getSubtoken(TokenStream& stream) { - if (pTok.current < pTok.data.size()) - return pTok.data[pTok.current++]; + if (stream.current < stream.data.size()) + return stream.data[stream.current++]; else return EndOfInput; } -void TPpContext::lUnreadByte(TokenStream& pTok) +// back up one position in the stream +void TPpContext::ungetSubtoken(TokenStream& stream) { - if (pTok.current > 0) - --pTok.current; + if (stream.current > 0) + --stream.current; } /* @@ -125,18 +126,15 @@ void TPpContext::RecordToken(TokenStream& pTok, int token, TPpToken* ppToken) const char* s; char* str = NULL; - if (token > PpAtomMaxSingle) - lAddByte(pTok, (unsigned char)((token & 0x7f) + 0x80)); - else - lAddByte(pTok, (unsigned char)(token & 0x7f)); + putSubtoken(pTok, token); switch (token) { case PpAtomIdentifier: case PpAtomConstString: s = ppToken->name; while (*s) - lAddByte(pTok, (unsigned char) *s++); - lAddByte(pTok, 0); + putSubtoken(pTok, *s++); + putSubtoken(pTok, 0); break; case PpAtomConstInt: case PpAtomConstUint: @@ -149,10 +147,10 @@ void TPpContext::RecordToken(TokenStream& pTok, int token, TPpToken* ppToken) #endif str = ppToken->name; while (*str) { - lAddByte(pTok, (unsigned char) *str); + putSubtoken(pTok, *str); str++; } - lAddByte(pTok, 0); + putSubtoken(pTok, 0); break; default: break; @@ -172,23 +170,21 @@ void TPpContext::RewindTokenStream(TokenStream& pTok) */ int TPpContext::ReadToken(TokenStream& pTok, TPpToken *ppToken) { - int ltoken, len; + int len; int ch; - ltoken = lReadByte(pTok); + int subtoken = getSubtoken(pTok); ppToken->loc = parseContext.getCurrentLoc(); - if (ltoken > 127) - ltoken += 128; - switch (ltoken) { + switch (subtoken) { case '#': // Check for ##, unless the current # is the last character if (pTok.current < pTok.data.size()) { - if (lReadByte(pTok) == '#') { + if (getSubtoken(pTok) == '#') { parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)"); parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)"); - ltoken = PpAtomPaste; + subtoken = PpAtomPaste; } else - lUnreadByte(pTok); + ungetSubtoken(pTok); } break; case PpAtomConstString: @@ -203,12 +199,12 @@ int TPpContext::ReadToken(TokenStream& pTok, TPpToken *ppToken) case PpAtomConstInt64: case PpAtomConstUint64: len = 0; - ch = lReadByte(pTok); + ch = getSubtoken(pTok); while (ch != 0 && ch != EndOfInput) { if (len < MaxTokenLength) { ppToken->name[len] = (char)ch; len++; - ch = lReadByte(pTok); + ch = getSubtoken(pTok); } else { parseContext.error(ppToken->loc, "token too long", "", ""); break; @@ -216,7 +212,7 @@ int TPpContext::ReadToken(TokenStream& pTok, TPpToken *ppToken) } ppToken->name[len] = 0; - switch (ltoken) { + switch (subtoken) { case PpAtomIdentifier: break; case PpAtomConstString: @@ -267,7 +263,7 @@ int TPpContext::ReadToken(TokenStream& pTok, TPpToken *ppToken) } } - return ltoken; + return subtoken; } int TPpContext::tTokenInput::scan(TPpToken* ppToken) @@ -285,14 +281,13 @@ bool TPpContext::tTokenInput::peekPasting() // 1. preceding ##? size_t savePos = tokens->current; - int byte; + int subtoken; // skip white space do { - byte = pp->lReadByte(*tokens); - } while (byte == ' '); - bool pasting = (byte == ((PpAtomPaste & 0x7f) + 0x80)); + subtoken = pp->getSubtoken(*tokens); + } while (subtoken == ' '); tokens->current = savePos; - if (pasting) + if (subtoken == PpAtomPaste) return true; // 2. last token and we've been told after this there will be a ## @@ -305,10 +300,10 @@ bool TPpContext::tTokenInput::peekPasting() savePos = tokens->current; bool moreTokens = false; do { - byte = pp->lReadByte(*tokens); - if (byte == EndOfInput) + subtoken = pp->getSubtoken(*tokens); + if (subtoken == EndOfInput) break; - if (byte != ' ') { + if (subtoken != ' ') { moreTokens = true; break; } diff --git a/glslang/MachineIndependent/preprocessor/PpTokens.h b/glslang/MachineIndependent/preprocessor/PpTokens.h index 3b1367f..9695c2f 100644 --- a/glslang/MachineIndependent/preprocessor/PpTokens.h +++ b/glslang/MachineIndependent/preprocessor/PpTokens.h @@ -82,7 +82,11 @@ namespace glslang { // Multi-character tokens enum EFixedAtoms { - PpAtomMaxSingle = 256, // single character tokens get their own char value as their token, skip them + // single character tokens get their own char value as their token; start here for multi-character tokens + PpAtomMaxSingle = 127, + + // replace bad character tokens with this, to avoid accidental aliasing with the below + PpAtomBadToken, // Operators -- 2.7.4