glsl/glcpp: Drop extra, final newline from most output
authorCarl Worth <cworth@cworth.org>
Fri, 20 Jun 2014 21:58:48 +0000 (14:58 -0700)
committerCarl Worth <cworth@cworth.org>
Tue, 29 Jul 2014 22:11:49 +0000 (15:11 -0700)
The glcpp parser is line-based, so it needs to see a NEWLINE token at the end
of each line. This causes a trick for files that end without a final newline.

Previously, the lexer for glcpp punted in this case by unconditionally
returning a NEWLINE token at end-of-file, (causing most files to have an extra
blank line at the end). Here, we refine this by lexing end-of-file as a
NEWLINE token only if the immediately preceding token was not a NEWLINE token.

The patch is a minor change that only looks huge for two reasons:

1. Almost all glcpp test result ".expected" files are updated to drop
   the extra newline.

2. All return statements from the lexer are adjusted to use a new
   RETURN_TOKEN macro that tracks the last-token-was-a-newline state.

Reviewed-by: Jordan Justen <jordan.l.justen@intel.com>
130 files changed:
src/glsl/glcpp/glcpp-lex.l
src/glsl/glcpp/glcpp-parse.y
src/glsl/glcpp/glcpp.h
src/glsl/glcpp/tests/000-content-with-spaces.c.expected
src/glsl/glcpp/tests/001-define.c.expected
src/glsl/glcpp/tests/002-define-chain.c.expected
src/glsl/glcpp/tests/003-define-chain-reverse.c.expected
src/glsl/glcpp/tests/004-define-recursive.c.expected
src/glsl/glcpp/tests/005-define-composite-chain.c.expected
src/glsl/glcpp/tests/006-define-composite-chain-reverse.c.expected
src/glsl/glcpp/tests/007-define-composite-recursive.c.expected
src/glsl/glcpp/tests/008-define-empty.c.expected
src/glsl/glcpp/tests/009-undef.c.expected
src/glsl/glcpp/tests/010-undef-re-define.c.expected
src/glsl/glcpp/tests/011-define-func-empty.c.expected
src/glsl/glcpp/tests/012-define-func-no-args.c.expected
src/glsl/glcpp/tests/013-define-func-1-arg-unused.c.expected
src/glsl/glcpp/tests/014-define-func-2-arg-unused.c.expected
src/glsl/glcpp/tests/015-define-object-with-parens.c.expected
src/glsl/glcpp/tests/016-define-func-1-arg.c.expected
src/glsl/glcpp/tests/017-define-func-2-args.c.expected
src/glsl/glcpp/tests/018-define-func-macro-as-parameter.c.expected
src/glsl/glcpp/tests/019-define-func-1-arg-multi.c.expected
src/glsl/glcpp/tests/020-define-func-2-arg-multi.c.expected
src/glsl/glcpp/tests/021-define-func-compose.c.expected
src/glsl/glcpp/tests/022-define-func-arg-with-parens.c.expected
src/glsl/glcpp/tests/023-define-extra-whitespace.c.expected
src/glsl/glcpp/tests/024-define-chain-to-self-recursion.c.expected
src/glsl/glcpp/tests/025-func-macro-as-non-macro.c.expected
src/glsl/glcpp/tests/026-define-func-extra-newlines.c.expected
src/glsl/glcpp/tests/027-define-chain-obj-to-func.c.expected
src/glsl/glcpp/tests/028-define-chain-obj-to-non-func.c.expected
src/glsl/glcpp/tests/029-define-chain-obj-to-func-with-args.c.expected
src/glsl/glcpp/tests/030-define-chain-obj-to-func-compose.c.expected
src/glsl/glcpp/tests/031-define-chain-func-to-func-compose.c.expected
src/glsl/glcpp/tests/032-define-func-self-recurse.c.expected
src/glsl/glcpp/tests/033-define-func-self-compose.c.expected
src/glsl/glcpp/tests/034-define-func-self-compose-non-func.c.expected
src/glsl/glcpp/tests/035-define-func-self-compose-non-func-multi-token-argument.c.expected
src/glsl/glcpp/tests/036-define-func-non-macro-multi-token-argument.c.expected
src/glsl/glcpp/tests/037-finalize-unexpanded-macro.c.expected
src/glsl/glcpp/tests/038-func-arg-with-commas.c.expected
src/glsl/glcpp/tests/039-func-arg-obj-macro-with-comma.c.expected
src/glsl/glcpp/tests/040-token-pasting.c.expected
src/glsl/glcpp/tests/041-if-0.c.expected
src/glsl/glcpp/tests/042-if-1.c.expected
src/glsl/glcpp/tests/043-if-0-else.c.expected
src/glsl/glcpp/tests/044-if-1-else.c.expected
src/glsl/glcpp/tests/045-if-0-elif.c.expected
src/glsl/glcpp/tests/046-if-1-elsif.c.expected
src/glsl/glcpp/tests/047-if-elif-else.c.expected
src/glsl/glcpp/tests/048-if-nested.c.expected
src/glsl/glcpp/tests/049-if-expression-precedence.c.expected
src/glsl/glcpp/tests/050-if-defined.c.expected
src/glsl/glcpp/tests/051-if-relational.c.expected
src/glsl/glcpp/tests/052-if-bitwise.c.expected
src/glsl/glcpp/tests/053-if-divide-and-shift.c.expected
src/glsl/glcpp/tests/054-if-with-macros.c.expected
src/glsl/glcpp/tests/055-define-chain-obj-to-func-parens-in-text.c.expected
src/glsl/glcpp/tests/056-macro-argument-with-comma.c.expected
src/glsl/glcpp/tests/057-empty-arguments.c.expected
src/glsl/glcpp/tests/058-token-pasting-empty-arguments.c.expected
src/glsl/glcpp/tests/059-token-pasting-integer.c.expected
src/glsl/glcpp/tests/060-left-paren-in-macro-right-paren-in-text.c.expected
src/glsl/glcpp/tests/061-define-chain-obj-to-func-multi.c.expected
src/glsl/glcpp/tests/062-if-0-skips-garbage.c.expected
src/glsl/glcpp/tests/063-comments.c.expected
src/glsl/glcpp/tests/064-version.c.expected
src/glsl/glcpp/tests/065-if-defined-parens.c.expected
src/glsl/glcpp/tests/066-if-nospace-expression.c.expected
src/glsl/glcpp/tests/067-nested-ifdef-ifndef.c.expected
src/glsl/glcpp/tests/068-accidental-pasting.c.expected
src/glsl/glcpp/tests/069-repeated-argument.c.expected
src/glsl/glcpp/tests/070-undefined-macro-in-expression.c.expected
src/glsl/glcpp/tests/071-punctuator.c.expected
src/glsl/glcpp/tests/072-token-pasting-same-line.c.expected
src/glsl/glcpp/tests/073-if-in-ifdef.c.expected
src/glsl/glcpp/tests/074-elif-undef.c.expected
src/glsl/glcpp/tests/075-elif-elif-undef.c.expected
src/glsl/glcpp/tests/076-elif-undef-nested.c.expected
src/glsl/glcpp/tests/077-else-without-if.c.expected
src/glsl/glcpp/tests/078-elif-without-if.c.expected
src/glsl/glcpp/tests/079-endif-without-if.c.expected
src/glsl/glcpp/tests/080-if-without-expression.c.expected
src/glsl/glcpp/tests/081-elif-without-expression.c.expected
src/glsl/glcpp/tests/082-invalid-paste.c.expected
src/glsl/glcpp/tests/083-unterminated-if.c.expected
src/glsl/glcpp/tests/085-incorrect-argument-count.c.expected
src/glsl/glcpp/tests/086-reserved-macro-names.c.expected
src/glsl/glcpp/tests/087-if-comments.c.expected
src/glsl/glcpp/tests/088-redefine-macro-legitimate.c.expected
src/glsl/glcpp/tests/089-redefine-macro-error.c.expected
src/glsl/glcpp/tests/090-hash-error.c.expected
src/glsl/glcpp/tests/091-hash-line.c.expected
src/glsl/glcpp/tests/092-redefine-macro-error-2.c.expected
src/glsl/glcpp/tests/093-divide-by-zero.c.expected
src/glsl/glcpp/tests/094-divide-by-zero-short-circuit.c.expected
src/glsl/glcpp/tests/095-recursive-define.c.expected
src/glsl/glcpp/tests/096-paste-twice.c.expected
src/glsl/glcpp/tests/097-paste-with-non-function-macro.c.expected
src/glsl/glcpp/tests/098-elif-undefined.c.expected
src/glsl/glcpp/tests/099-c99-example.c.expected
src/glsl/glcpp/tests/100-macro-with-colon.c.expected
src/glsl/glcpp/tests/101-macros-used-twice.c.expected
src/glsl/glcpp/tests/104-hash-line-followed-by-code.c.expected
src/glsl/glcpp/tests/105-multiline-hash-line.c.expected
src/glsl/glcpp/tests/106-multiline-hash-if.c.expected
src/glsl/glcpp/tests/107-multiline-hash-elif.c.expected
src/glsl/glcpp/tests/108-no-space-after-hash-version.c.expected
src/glsl/glcpp/tests/109-no-space-after-hash-line.c.expected
src/glsl/glcpp/tests/110-no-space-digits-after-hash-elif.c.expected
src/glsl/glcpp/tests/111-no-space-operator-after-hash-if.c.expected
src/glsl/glcpp/tests/112-no-space-operator-after-hash-elif.c.expected
src/glsl/glcpp/tests/113-line-and-file-macros.c.expected
src/glsl/glcpp/tests/114-paste-integer-tokens.c.expected
src/glsl/glcpp/tests/115-line-continuations.c.expected
src/glsl/glcpp/tests/116-disable-line-continuations.c.expected
src/glsl/glcpp/tests/117-line-continuation-and-non-continuation-backslash.c.expected
src/glsl/glcpp/tests/118-comment-becomes-space.c.expected
src/glsl/glcpp/tests/118-multiple-else.c.expected
src/glsl/glcpp/tests/119-elif-after-else.c.expected
src/glsl/glcpp/tests/120-undef-builtin.c.expected
src/glsl/glcpp/tests/121-comment-bug-72686.c.expected
src/glsl/glcpp/tests/122-redefine-whitespace.c.expected
src/glsl/glcpp/tests/124-preprocessing-numbers.c.expected
src/glsl/glcpp/tests/125-es-short-circuit-undefined.c.expected
src/glsl/glcpp/tests/126-garbage-after-directive.c.expected
src/glsl/glcpp/tests/127-pragma-empty.c.expected
src/glsl/glcpp/tests/128-space-before-hash.c.expected
src/glsl/glcpp/tests/130-define-comment.c.expected

index 44547fd..980ab5d 100644 (file)
@@ -60,6 +60,16 @@ void glcpp_set_column (int  column_no , yyscan_t yyscanner);
                yycolumn = 1;           \
                yylloc->source = 0;     \
        } while(0)
+
+#define RETURN_TOKEN(token)                                    \
+       do {                                                    \
+               if (token == NEWLINE)                           \
+                       parser->last_token_was_newline = 1;     \
+               else                                            \
+                       parser->last_token_was_newline = 0;     \
+               return (token);                                 \
+       } while(0)
+
 %}
 
 %option bison-bridge bison-locations reentrant noyywrap
@@ -111,7 +121,7 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
                        parser->commented_newlines--;
                if (parser->commented_newlines == 0)
                        BEGIN INITIAL;
-               return NEWLINE;
+               RETURN_TOKEN (NEWLINE);
        }
 
        /* The handling of the SKIP vs INITIAL start states requires
@@ -169,13 +179,13 @@ HEXADECIMAL_INTEGER       0[xX][0-9a-fA-F]+[uU]?
 <COMMENT>"*"+"/"        {
        yy_pop_state(yyscanner);
        if (yyextra->space_tokens)
-               return SPACE;
+               RETURN_TOKEN (SPACE);
 }
 
 {HASH}version{HSPACE}+ {
        yylval->str = ralloc_strdup (yyextra, yytext);
        yyextra->space_tokens = 0;
-       return HASH_VERSION;
+       RETURN_TOKEN (HASH_VERSION);
 }
 
        /* glcpp doesn't handle #extension, #version, or #pragma directives.
@@ -184,46 +194,46 @@ HEXADECIMAL_INTEGER       0[xX][0-9a-fA-F]+[uU]?
        yylval->str = ralloc_strdup (yyextra, yytext);
        yylineno++;
        yycolumn = 0;
-       return OTHER;
+       RETURN_TOKEN (OTHER);
 }
 
 {HASH}line{HSPACE}+ {
-       return HASH_LINE;
+       RETURN_TOKEN (HASH_LINE);
 }
 
 <SKIP,INITIAL>{
 {HASH}ifdef {
        yyextra->lexing_directive = 1;
        yyextra->space_tokens = 0;
-       return HASH_IFDEF;
+       RETURN_TOKEN (HASH_IFDEF);
 }
 
 {HASH}ifndef {
        yyextra->lexing_directive = 1;
        yyextra->space_tokens = 0;
-       return HASH_IFNDEF;
+       RETURN_TOKEN (HASH_IFNDEF);
 }
 
 {HASH}if/[^_a-zA-Z0-9] {
        yyextra->lexing_directive = 1;
        yyextra->space_tokens = 0;
-       return HASH_IF;
+       RETURN_TOKEN (HASH_IF);
 }
 
 {HASH}elif/[^_a-zA-Z0-9] {
        yyextra->lexing_directive = 1;
        yyextra->space_tokens = 0;
-       return HASH_ELIF;
+       RETURN_TOKEN (HASH_ELIF);
 }
 
 {HASH}else {
        yyextra->space_tokens = 0;
-       return HASH_ELSE;
+       RETURN_TOKEN (HASH_ELSE);
 }
 
 {HASH}endif {
        yyextra->space_tokens = 0;
-       return HASH_ENDIF;
+       RETURN_TOKEN (HASH_ENDIF);
 }
 }
 
@@ -258,21 +268,21 @@ HEXADECIMAL_INTEGER       0[xX][0-9a-fA-F]+[uU]?
 {HASH}define{HSPACE}+ {
        yyextra->space_tokens = 0;
        yy_push_state(DEFINE, yyscanner);
-       return HASH_DEFINE;
+       RETURN_TOKEN (HASH_DEFINE);
 }
 
        /* An identifier immediately followed by '(' */
 <DEFINE>{IDENTIFIER}/"(" {
        yy_pop_state(yyscanner);
        yylval->str = ralloc_strdup (yyextra, yytext);
-       return FUNC_IDENTIFIER;
+       RETURN_TOKEN (FUNC_IDENTIFIER);
 }
 
        /* An identifier not immediately followed by '(' */
 <DEFINE>{IDENTIFIER} {
        yy_pop_state(yyscanner);
        yylval->str = ralloc_strdup (yyextra, yytext);
-       return OBJ_IDENTIFIER;
+       RETURN_TOKEN (OBJ_IDENTIFIER);
 }
 
        /* Whitespace */
@@ -284,7 +294,7 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
 <DEFINE>[/][^*]{NONSPACE}* {
        BEGIN INITIAL;
        glcpp_error(yylloc, yyextra, "#define followed by a non-identifier: %s", yytext);
-       return INTEGER_STRING;
+       RETURN_TOKEN (INTEGER_STRING);
 }
 
        /* A character that can't start an identifier, comment, or
@@ -292,98 +302,98 @@ HEXADECIMAL_INTEGER       0[xX][0-9a-fA-F]+[uU]?
 <DEFINE>[^_a-zA-Z/[:space:]]{NONSPACE}* {
        BEGIN INITIAL;
        glcpp_error(yylloc, yyextra, "#define followed by a non-identifier: %s", yytext);
-       return INTEGER_STRING;
+       RETURN_TOKEN (INTEGER_STRING);
 }
 
 {HASH}undef {
        yyextra->space_tokens = 0;
-       return HASH_UNDEF;
+       RETURN_TOKEN (HASH_UNDEF);
 }
 
 {HASH} {
        yyextra->space_tokens = 0;
-       return HASH;
+       RETURN_TOKEN (HASH);
 }
 
 {DECIMAL_INTEGER} {
        yylval->str = ralloc_strdup (yyextra, yytext);
-       return INTEGER_STRING;
+       RETURN_TOKEN (INTEGER_STRING);
 }
 
 {OCTAL_INTEGER} {
        yylval->str = ralloc_strdup (yyextra, yytext);
-       return INTEGER_STRING;
+       RETURN_TOKEN (INTEGER_STRING);
 }
 
 {HEXADECIMAL_INTEGER} {
        yylval->str = ralloc_strdup (yyextra, yytext);
-       return INTEGER_STRING;
+       RETURN_TOKEN (INTEGER_STRING);
 }
 
 "<<"  {
-       return LEFT_SHIFT;
+       RETURN_TOKEN (LEFT_SHIFT);
 }
 
 ">>" {
-       return RIGHT_SHIFT;
+       RETURN_TOKEN (RIGHT_SHIFT);
 }
 
 "<=" {
-       return LESS_OR_EQUAL;
+       RETURN_TOKEN (LESS_OR_EQUAL);
 }
 
 ">=" {
-       return GREATER_OR_EQUAL;
+       RETURN_TOKEN (GREATER_OR_EQUAL);
 }
 
 "==" {
-       return EQUAL;
+       RETURN_TOKEN (EQUAL);
 }
 
 "!=" {
-       return NOT_EQUAL;
+       RETURN_TOKEN (NOT_EQUAL);
 }
 
 "&&" {
-       return AND;
+       RETURN_TOKEN (AND);
 }
 
 "||" {
-       return OR;
+       RETURN_TOKEN (OR);
 }
 
 "##" {
        if (parser->is_gles)
                glcpp_error(yylloc, yyextra, "Token pasting (##) is illegal in GLES");
-       return PASTE;
+       RETURN_TOKEN (PASTE);
 }
 
 "defined" {
-       return DEFINED;
+       RETURN_TOKEN (DEFINED);
 }
 
 {IDENTIFIER} {
        yylval->str = ralloc_strdup (yyextra, yytext);
-       return IDENTIFIER;
+       RETURN_TOKEN (IDENTIFIER);
 }
 
 {PP_NUMBER} {
        yylval->str = ralloc_strdup (yyextra, yytext);
-       return OTHER;
+       RETURN_TOKEN (OTHER);
 }
 
 {PUNCTUATION} {
-       return yytext[0];
+       RETURN_TOKEN (yytext[0]);
 }
 
 {OTHER}+ {
        yylval->str = ralloc_strdup (yyextra, yytext);
-       return OTHER;
+       RETURN_TOKEN (OTHER);
 }
 
 {HSPACE} {
        if (yyextra->space_tokens) {
-               return SPACE;
+               RETURN_TOKEN (SPACE);
        }
 }
 
@@ -395,7 +405,7 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
        yyextra->lexing_directive = 0;
        yylineno++;
        yycolumn = 0;
-       return NEWLINE;
+       RETURN_TOKEN (NEWLINE);
 }
 
 <INITIAL,COMMENT,DEFINE><<EOF>> {
@@ -405,7 +415,8 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
                glcpp_error(yylloc, yyextra, "#define without macro name");
        BEGIN DONE; /* Don't keep matching this rule forever. */
        yyextra->lexing_directive = 0;
-       return NEWLINE;
+       if (! parser->last_token_was_newline)
+               RETURN_TOKEN (NEWLINE);
 }
 
        /* We don't actually use the UNREACHABLE start condition. We
index 084078e..07d780e 100644 (file)
@@ -1308,6 +1308,7 @@ glcpp_parser_create (const struct gl_extensions *extensions, gl_api api)
        parser->active = NULL;
        parser->lexing_directive = 0;
        parser->space_tokens = 1;
+        parser->last_token_was_newline = 0;
        parser->newline_as_space = 0;
        parser->in_control_line = 0;
        parser->paren_count = 0;
index 64b4872..6316c9f 100644 (file)
@@ -177,6 +177,7 @@ struct glcpp_parser {
        active_list_t *active;
        int lexing_directive;
        int space_tokens;
+       int last_token_was_newline;
        int newline_as_space;
        int in_control_line;
        int paren_count;
index 2dd21c0..b48a2d6 100644 (file)
@@ -17,4 +17,3 @@ Pasting "4" and "+" does not give a valid preprocessing token.
 2
 34
 45.2
-
index 19be750..352bbff 100644 (file)
@@ -14,4 +14,3 @@
 f(2 * (y+1)) + f(2 * (f(2 * (z[0])))) % f(2 * (0)) + t(1);
 f(2 * (2 +(3,4)-0,1)) | f(2 * (~ 5)) & f(2 * (0,1))^m(0,1);
 int i[] = { 1, 23, 4, 5, };
-
index 3562fb9..55bc788 100644 (file)
@@ -5,4 +5,3 @@
 5. Hexadecimal for 20560: 5 0 5 0
 6: Zip code for Nortonville, KS: 6 6 0 6 0
 7. James Bond, as a number: 0 0 7
-
index a3ad7da..aa97110 100644 (file)
@@ -5,4 +5,3 @@
 5. Hexadecimal for 20560: 5050
 6: Zip code for Nortonville, KS: 66060
 7. James Bond, as a number: 007
-