[flang] Strip comments from source and directive lines after macro replacement

Original-commit: flang-compiler/f18@ecf22e6698e82b36e5d709aa0362ac9f7591ea89
Reviewed-on: https://github.com/flang-compiler/f18/pull/460

diff --git a/flang/lib/parser/char-block.h b/flang/lib/parser/char-block.h
index e9439f4..24c5266 100644 (file)
--- a/flang/lib/parser/char-block.h
+++ b/flang/lib/parser/char-block.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+// Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -56,15 +56,17 @@ public:
     interval_.ExtendToCover(that.interval_);
   }
 
-  bool IsBlank() const {
+  char FirstNonBlank() const {
     for (char ch : *this) {
       if (ch != ' ' && ch != '\t') {
-        return false;
+        return ch;
       }
     }
-    return true;
+    return ' ';  // non no-blank character
   }
 
+  bool IsBlank() const { return FirstNonBlank() == ' '; }
+
   std::string ToString() const {
     return std::string{interval_.start(), interval_.size()};
   }

diff --git a/flang/lib/parser/prescan.cc b/flang/lib/parser/prescan.cc
index c8761c6..bf7afb1 100644 (file)
--- a/flang/lib/parser/prescan.cc
+++ b/flang/lib/parser/prescan.cc
@@ -193,7 +193,7 @@ void Prescanner::Statement() {
       NormalizeCompilerDirectiveCommentMarker(*preprocessed);
       preprocessed->ToLowerCase();
       SourceFormChange(preprocessed->ToString());
-      preprocessed->Emit(cooked_);
+      preprocessed->ClipComment().Emit(cooked_);
       break;
     case LineClassification::Kind::Source:
       if (inFixedForm_) {
@@ -205,7 +205,7 @@ void Prescanner::Statement() {
           preprocessed->RemoveRedundantBlanks();
         }
       }
-      preprocessed->ToLowerCase().Emit(cooked_);
+      preprocessed->ToLowerCase().ClipComment().Emit(cooked_);
       break;
     }
   } else {

diff --git a/flang/lib/parser/token-sequence.cc b/flang/lib/parser/token-sequence.cc
index eb9e8d8..a79a048 100644 (file)
--- a/flang/lib/parser/token-sequence.cc
+++ b/flang/lib/parser/token-sequence.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+// Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -221,6 +221,21 @@ TokenSequence &TokenSequence::RemoveRedundantBlanks(std::size_t firstChar) {
   return *this;
 }
 
+TokenSequence &TokenSequence::ClipComment() {
+  std::size_t tokens{SizeInTokens()};
+  for (std::size_t j{0}; j < tokens; ++j) {
+    if (TokenAt(j).FirstNonBlank() == '!') {
+      TokenSequence result;
+      if (j > 0) {
+        result.Put(*this, 0, j - 1);
+      }
+      swap(result);
+      return *this;
+    }
+  }
+  return *this;
+}
+
 void TokenSequence::Emit(CookedSource &cooked) const {
   cooked.Put(&char_[0], char_.size());
   cooked.PutProvenanceMappings(provenances_);

diff --git a/flang/lib/parser/token-sequence.h b/flang/lib/parser/token-sequence.h
index 0ba6a6d..e4e2a7d 100644 (file)
--- a/flang/lib/parser/token-sequence.h
+++ b/flang/lib/parser/token-sequence.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+// Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -93,6 +93,7 @@ public:
   void Put(const CharBlock &, Provenance);
   void Put(const std::string &, Provenance);
   void Put(const std::stringstream &, Provenance);
+
   Provenance GetTokenProvenance(
       std::size_t token, std::size_t offset = 0) const;
   ProvenanceRange GetTokenProvenanceRange(
@@ -107,6 +108,7 @@ public:
   bool HasRedundantBlanks(std::size_t firstChar = 0) const;
   TokenSequence &RemoveBlanks(std::size_t firstChar = 0);
   TokenSequence &RemoveRedundantBlanks(std::size_t firstChar = 0);
+  TokenSequence &ClipComment();
   void Emit(CookedSource &) const;
   void Dump(std::ostream &) const;