Fix mishandling of escaped newlines followed by newlines or nuls.

Previously, if an escaped newline was followed by a newline or a nul, we'd lex
the escaped newline as a bogus space character. This led to a bunch of
different broken corner cases:

For the pattern "\\\n\0#", we would then have a (horizontal) space whose
spelling ends in a newline, and would decide that the '#' is at the start of a
line, and incorrectly start preprocessing a directive in the middle of a
logical source line. If we were already in the middle of a directive, this
would result in our attempting to process multiple directives at the same time!
This resulted in crashes, asserts, and hangs on invalid input, as discovered by
fuzz-testing.

For the pattern "\\\n" at EOF (with an implicit following nul byte), we would
produce a bogus trailing space character with spelling "\\\n". This was mostly
harmless, but would lead to clang-format getting confused and misformatting in
rare cases. We now produce a trailing EOF token with spelling "\\\n",
consistent with our handling for other similar cases -- an escaped newline is
always part of the token containing the next character, if any.

For the pattern "\\\n\n", this was somewhat more benign, but would produce an
extraneous whitespace token to clients who care about preserving whitespace.
However, it turns out that our lexing for line comments was relying on this bug
due to an off-by-one error in its computation of the end of the comment, on the
slow path where the comment might contain escaped newlines.

llvm-svn: 300515

diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index 4ee43d6937e00866b10d1dcff180afadf9859bb4..1acc0c30651263a833f46106913110b63ec0aca0 100644 (file)
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -467,6 +467,9 @@ FormatToken *FormatTokenLexer::getNextToken() {
       if (pos >= 0 && Text[pos] == '\r')
         --pos;
       // See whether there is an odd number of '\' before this.
+      // FIXME: This is wrong. A '\' followed by a newline is always removed,
+      // regardless of whether there is another '\' before it.
+      // FIXME: Newlines can also be escaped by a '?' '?' '/' trigraph.
       unsigned count = 0;
       for (; pos >= 0; --pos, ++count)
         if (Text[pos] != '\\')

diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 4c051939471c76ff8a71bbbb474796aa101e11c1..5d0fe42defdde9277493a9dd22d7969dc44a4f27 100644 (file)
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -1282,12 +1282,6 @@ Slash:
       Size += EscapedNewLineSize;
       Ptr  += EscapedNewLineSize;
 
-      // If the char that we finally got was a \n, then we must have had
-      // something like \<newline><newline>.  We don't want to consume the
-      // second newline.
-      if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0')
-        return ' ';
-
       // Use slow version to accumulate a correct size field.
       return getCharAndSizeSlow(Ptr, Size, Tok);
     }
@@ -1338,12 +1332,6 @@ Slash:
       Size += EscapedNewLineSize;
       Ptr  += EscapedNewLineSize;
 
-      // If the char that we finally got was a \n, then we must have had
-      // something like \<newline><newline>.  We don't want to consume the
-      // second newline.
-      if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0')
-        return ' ';
-
       // Use slow version to accumulate a correct size field.
       return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts);
     }
@@ -2070,8 +2058,11 @@ bool Lexer::SkipLineComment(Token &Result, const char *CurPtr,
   // Scan over the body of the comment.  The common case, when scanning, is that
   // the comment contains normal ascii characters with nothing interesting in
   // them.  As such, optimize for this case with the inner loop.
+  //
+  // This loop terminates with CurPtr pointing at the newline (or end of buffer)
+  // character that ends the line comment.
   char C;
-  do {
+  while (true) {
     C = *CurPtr;
     // Skip over characters in the fast loop.
     while (C != 0 &&                // Potentially EOF.
@@ -2097,6 +2088,8 @@ bool Lexer::SkipLineComment(Token &Result, const char *CurPtr,
         break; // This is a newline, we're done.
 
       // If there was space between the backslash and newline, warn about it.
+      // FIXME: This warning is bogus if trigraphs are disabled and the line
+      // ended with '?' '?' '\\' '\n'.
       if (HasSpace && !isLexingRawMode())
         Diag(EscapePtr, diag::backslash_newline_space);
     }
@@ -2140,9 +2133,9 @@ bool Lexer::SkipLineComment(Token &Result, const char *CurPtr,
         }
     }
 
-    if (CurPtr == BufferEnd+1) { 
-      --CurPtr; 
-      break; 
+    if (C == '\r' || C == '\n' || CurPtr == BufferEnd + 1) {
+      --CurPtr;
+      break;
     }
 
     if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) {
@@ -2150,8 +2143,7 @@ bool Lexer::SkipLineComment(Token &Result, const char *CurPtr,
       cutOffLexing();
       return false;
     }
-
-  } while (C != '\n' && C != '\r');
+  }
 
   // Found but did not consume the newline.  Notify comment handlers about the
   // comment unless we're in a #if 0 block.

diff --git a/clang/test/Lexer/newline-nul.c b/clang/test/Lexer/newline-nul.c
new file mode 100644 (file)
index 0000000..dd1b419
Binary files /dev/null and b/clang/test/Lexer/newline-nul.c differ

diff --git a/clang/unittests/Format/FormatTestComments.cpp b/clang/unittests/Format/FormatTestComments.cpp
index df24abe2612575373ba658cb73d99f7d030af273..e6310158e91e9dbe5721038e75761c037d0fda9b 100644 (file)
--- a/clang/unittests/Format/FormatTestComments.cpp
+++ b/clang/unittests/Format/FormatTestComments.cpp
@@ -2018,7 +2018,7 @@ TEST_F(FormatTestComments, AlignTrailingComments) {
             format("#define MACRO(V)\\\n"
                    "V(Rt2)  /* one more char */ \\\n"
                    "V(Rs) /* than here  */    \\\n"
-                   "/* comment 3 */         \\\n",
+                   "/* comment 3 */\n",
                    getLLVMStyleWithColumns(40)));
   EXPECT_EQ("int i = f(abc, // line 1\n"
             "          d,   // line 2\n"