[flang] Forge provenances for compiler-inserted text.

Original-commit: flang-compiler/f18@dbdd01aa258557d45816d1a1e0588553c16bef6a
Reviewed-on: https://github.com/flang-compiler/f18/pull/9
Tree-same-pre-rewrite: false

diff --git a/flang/lib/parser/preprocessor.cc b/flang/lib/parser/preprocessor.cc
index 4b703cb..b4fdbf0 100644 (file)
--- a/flang/lib/parser/preprocessor.cc
+++ b/flang/lib/parser/preprocessor.cc
@@ -132,8 +132,9 @@ Definition::Definition(const std::vector<std::string> &argNames,
     argumentCount_(argNames.size()), isVariadic_{isVariadic},
     replacement_{Tokenize(argNames, repl, firstToken, tokens)} {}
 
-Definition::Definition(const std::string &predefined)
-  : isPredefined_{true}, replacement_{predefined} {}
+Definition::Definition(const std::string &predefined, AllSources *sources)
+  : isPredefined_{true}, replacement_{predefined,
+                             sources->AddCompilerInsertion(predefined).start} {}
 
 bool Definition::set_isDisabled(bool disable) {
   bool was{isDisabled_};
@@ -172,7 +173,8 @@ TokenSequence Definition::Tokenize(const std::vector<std::string> &argNames,
   return result;
 }
 
-TokenSequence Definition::Apply(const std::vector<TokenSequence> &args) {
+TokenSequence Definition::Apply(
+    const std::vector<TokenSequence> &args, const Prescanner &prescanner) {
   TokenSequence result;
   bool pasting{false};
   bool skipping{false};
@@ -208,7 +210,8 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args) {
         while (result.size() >= afterLastNonBlank) {
           result.pop_back();
         }
-        result.PutNextTokenChar('"', 0);  // TODO provenance
+        result.PutNextTokenChar(
+            '"', prescanner.CompilerInsertionProvenance('"'));
         for (size_t k{0}; k < argTokens; ++k) {
           const CharPointerWithLength &arg{args[index][k]};
           size_t argBytes{args[index][k].size()};
@@ -221,7 +224,8 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args) {
             result.PutNextTokenChar(ch, from);
           }
         }
-        result.PutNextTokenChar('"', 0);  // TODO provenance
+        result.PutNextTokenChar(
+            '"', prescanner.CompilerInsertionProvenance('"'));
         result.CloseToken();
       } else {
         for (size_t k{0}; k < argTokens; ++k) {
@@ -247,7 +251,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args) {
         token.ToString() == "__VA_ARGS__") {
       for (size_t k{argumentCount_}; k < args.size(); ++k) {
         if (k > argumentCount_) {
-          result.Put(","s, 0);  // TODO provenance
+          result.Put(","s, prescanner.CompilerInsertionProvenance(','));
         }
         result.Put(args[k]);
       }
@@ -283,13 +287,15 @@ Preprocessor::Preprocessor(Prescanner &ps) : prescanner_{ps} {
   // of __DATE__ or __TIME__ change during compilation.
   std::time_t now;
   std::time(&now);
-  definitions_.emplace(SaveToken("__DATE__"s),  // e.g., "Jun 16 1904"
-      Definition{FormatTime(now, "\"%h %e %Y\""), 0, 1});
-  definitions_.emplace(SaveToken("__TIME__"s),  // e.g., "23:59:60"
-      Definition{FormatTime(now, "\"%T\""), 0, 1});
+  definitions_.emplace(SaveTokenAsName("__DATE__"s),  // e.g., "Jun 16 1904"
+      Definition{FormatTime(now, "\"%h %e %Y\""), ps.allSources()});
+  definitions_.emplace(SaveTokenAsName("__TIME__"s),  // e.g., "23:59:60"
+      Definition{FormatTime(now, "\"%T\""), ps.allSources()});
   // The values of these predefined macros depend on their invocation sites.
-  definitions_.emplace(SaveToken("__FILE__"s), Definition{"__FILE__"s});
-  definitions_.emplace(SaveToken("__LINE__"s), Definition{"__LINE__"s});
+  definitions_.emplace(
+      SaveTokenAsName("__FILE__"s), Definition{"__FILE__"s, ps.allSources()});
+  definitions_.emplace(
+      SaveTokenAsName("__LINE__"s), Definition{"__LINE__"s, ps.allSources()});
 }
 
 bool Preprocessor::MacroReplacement(
@@ -328,17 +334,21 @@ bool Preprocessor::MacroReplacement(
       if (def.isPredefined()) {
         std::string name{def.replacement()[0].ToString()};
         if (name == "__FILE__") {
-          result->Put("\""s +
-              prescanner_.allSources().GetPath(
+          std::string f{"\""s +
+              prescanner_.allSources()->GetPath(
                   prescanner_.GetCurrentProvenance()) +
-              '"');
+              '"'};
+          result->Put(
+              f, prescanner_.allSources()->AddCompilerInsertion(f).start);
           continue;
         }
         if (name == "__LINE__") {
           std::stringstream ss;
-          ss << prescanner_.allSources().GetLineNumber(
+          ss << prescanner_.allSources()->GetLineNumber(
               prescanner_.GetCurrentProvenance());
-          result->Put(ss.str());
+          std::string s{ss.str()};
+          result->Put(
+              s, prescanner_.allSources()->AddCompilerInsertion(s).start);
           continue;
         }
       }
@@ -391,7 +401,7 @@ bool Preprocessor::MacroReplacement(
       args.emplace_back(TokenSequence(input, at, count));
     }
     def.set_isDisabled(true);
-    result->Put(ReplaceMacros(def.Apply(args)));
+    result->Put(ReplaceMacros(def.Apply(args, prescanner_)));
     def.set_isDisabled(false);
   }
   return true;
@@ -478,7 +488,7 @@ bool Preprocessor::Directive(const TokenSequence &dir) {
       Complain("#define: missing or invalid name");
       return false;
     }
-    nameToken = SaveToken(nameToken);
+    nameToken = SaveTokenAsName(nameToken);
     definitions_.erase(nameToken);
     if (++j < tokens && dir[j].size() == 1 && dir[j][0] == '(') {
       j = SkipBlanks(dir, j + 1, tokens);
@@ -615,7 +625,8 @@ bool Preprocessor::Directive(const TokenSequence &dir) {
   return false;
 }
 
-CharPointerWithLength Preprocessor::SaveToken(const CharPointerWithLength &t) {
+CharPointerWithLength Preprocessor::SaveTokenAsName(
+    const CharPointerWithLength &t) {
   names_.push_back(t.ToString());
   return {names_.back().data(), names_.back().size()};
 }
@@ -652,7 +663,7 @@ bool Preprocessor::SkipDisabledConditionalCode(
 }
 
 void Preprocessor::Complain(const std::string &message) {
-  prescanner_.messages().Put({prescanner_.GetCurrentProvenance(), message});
+  prescanner_.messages()->Put({prescanner_.GetCurrentProvenance(), message});
 }
 
 // Precedence level codes used here to accommodate mixed Fortran and C:
@@ -661,7 +672,7 @@ void Preprocessor::Complain(const std::string &message) {
 // 13: **
 // 12: *, /, % (modulus)
 // 11: + and -
-//  0: << and >>
+// 10: << and >>
 //  9: bitwise &
 //  8: bitwise ^
 //  7: bitwise |
@@ -942,7 +953,8 @@ bool Preprocessor::IsIfPredicateTrue(
         name = expr1[j++];
       }
       if (!name.empty()) {
-        expr2.Put(IsNameDefined(name) ? "1" : "0", 1, 0);  // TODO provenance
+        char truth{IsNameDefined(name) ? '1' : '0'};
+        expr2.Put(&truth, 1, prescanner_.CompilerInsertionProvenance(truth));
         continue;
       }
     }

diff --git a/flang/lib/parser/preprocessor.h b/flang/lib/parser/preprocessor.h
index c799f0d..b30602c 100644 (file)
--- a/flang/lib/parser/preprocessor.h
+++ b/flang/lib/parser/preprocessor.h
@@ -91,7 +91,7 @@ public:
     : start_{std::move(that.start_)}, nextStart_{that.nextStart_},
       char_{std::move(that.char_)}, provenances_{std::move(that.provenances_)} {
   }
-  TokenSequence(const std::string &s) { Put(s, 0); }  // TODO predefined prov.
+  TokenSequence(const std::string &s, Provenance p) { Put(s, p); }
 
   TokenSequence &operator=(const TokenSequence &that) {
     clear();
@@ -159,7 +159,7 @@ public:
   Definition(const TokenSequence &, size_t firstToken, size_t tokens);
   Definition(const std::vector<std::string> &argNames, const TokenSequence &,
       size_t firstToken, size_t tokens, bool isVariadic = false);
-  explicit Definition(const std::string &predefined);
+  Definition(const std::string &predefined, AllSources *);
 
   bool isFunctionLike() const { return isFunctionLike_; }
   size_t argumentCount() const { return argumentCount_; }
@@ -170,7 +170,8 @@ public:
 
   bool set_isDisabled(bool disable);
 
-  TokenSequence Apply(const std::vector<TokenSequence> &args);
+  TokenSequence Apply(
+      const std::vector<TokenSequence> &args, const Prescanner &);
 
 private:
   static TokenSequence Tokenize(const std::vector<std::string> &argNames,
@@ -203,7 +204,7 @@ private:
   enum class CanDeadElseAppear { No, Yes };
 
   void Complain(const std::string &);
-  CharPointerWithLength SaveToken(const CharPointerWithLength &);
+  CharPointerWithLength SaveTokenAsName(const CharPointerWithLength &);
   bool IsNameDefined(const CharPointerWithLength &);
   TokenSequence ReplaceMacros(const TokenSequence &);
   bool SkipDisabledConditionalCode(const std::string &dirName, IsElseActive);

diff --git a/flang/lib/parser/prescan.cc b/flang/lib/parser/prescan.cc
index 518471a..139ddf2 100644 (file)
--- a/flang/lib/parser/prescan.cc
+++ b/flang/lib/parser/prescan.cc
@@ -9,14 +9,23 @@
 namespace Fortran {
 namespace parser {
 
-CookedSource Prescanner::Prescan(AllSources *allSources) {
+Prescanner::Prescanner(Messages *messages, AllSources *allSources)
+  : messages_{messages}, allSources_{allSources}, preprocessor_{*this} {
+  std::string compilerInserts{" ,\"01"};
+  ProvenanceRange range{allSources->AddCompilerInsertion(compilerInserts)};
+  for (size_t j{0}; j < compilerInserts.size(); ++j) {
+    compilerInsertionProvenance_[compilerInserts[j]] = range.start + j;
+  }
+}
+
+CookedSource Prescanner::Prescan() {
   startProvenance_ = 0;
-  start_ = &(*allSources)[0];
-  limit_ = start_ + allSources->size();
+  start_ = &(*allSources_)[0];
+  limit_ = start_ + allSources_->size();
   lineStart_ = start_;
   BeginSourceLine(start_);
   TokenSequence tokens, preprocessed;
-  CookedSource cooked{allSources};
+  CookedSource cooked{allSources_};
   while (lineStart_ < limit_) {
     if (CommentLinesAndPreprocessorDirectives() && lineStart_ >= limit_) {
       break;
@@ -68,6 +77,10 @@ std::optional<TokenSequence> Prescanner::NextTokenizedLine() {
   return {std::move(tokens)};
 }
 
+Provenance Prescanner::CompilerInsertionProvenance(char ch) const {
+  return compilerInsertionProvenance_.find(ch)->second;
+}
+
 void Prescanner::NextLine() {
   void *vstart{static_cast<void *>(const_cast<char *>(lineStart_))};
   void *v{std::memchr(vstart, '\n', limit_ - lineStart_)};
@@ -98,8 +111,9 @@ void Prescanner::LabelField(TokenSequence *token) {
     token->CloseToken();
   }
   if (outCol < 7) {
+    Provenance provenance{CompilerInsertionProvenance(' ')};
     for (; outCol < 7; ++outCol) {
-      EmitChar(token, ' ');
+      token->PutNextTokenChar(' ', provenance);
     }
     token->CloseToken();
   }
@@ -192,7 +206,7 @@ bool Prescanner::NextToken(TokenSequence *tokens) {
       inCharLiteral_ = true;
       while (n-- > 0) {
         if (PadOutCharacterLiteral()) {
-          EmitChar(tokens, ' ');
+          tokens->PutNextTokenChar(' ', compilerInsertionProvenance_[' ']);
         } else {
           if (*at_ == '\n') {
             break;  // TODO error
@@ -283,12 +297,12 @@ void Prescanner::QuotedCharacterLiteral(TokenSequence *tokens) {
   do {
     EmitCharAndAdvance(tokens, *at_);
     while (PadOutCharacterLiteral()) {
-      EmitChar(tokens, ' ');
+      tokens->PutNextTokenChar(' ', compilerInsertionProvenance_[' ']);
     }
     if (*at_ == '\\' && enableBackslashEscapesInCharLiterals_) {
       EmitCharAndAdvance(tokens, '\\');
       while (PadOutCharacterLiteral()) {
-        EmitChar(tokens, ' ');
+        tokens->PutNextTokenChar(' ', compilerInsertionProvenance_[' ']);
       }
     } else if (*at_ == quote) {
       // A doubled quote mark becomes a single instance of the quote character

diff --git a/flang/lib/parser/prescan.h b/flang/lib/parser/prescan.h
index 819e394..a4c22b3 100644 (file)
--- a/flang/lib/parser/prescan.h
+++ b/flang/lib/parser/prescan.h
@@ -12,6 +12,7 @@
 #include "preprocessor.h"
 #include "provenance.h"
 #include "source.h"
+#include <map>
 #include <optional>
 #include <string>
 
@@ -20,10 +21,9 @@ namespace parser {
 
 class Prescanner {
 public:
-  explicit Prescanner(Messages &messages)
-    : messages_{messages}, preprocessor_{*this} {}
+  Prescanner(Messages *, AllSources *);
 
-  Messages &messages() const { return messages_; }
+  Messages *messages() const { return messages_; }
   bool anyFatalErrors() const { return anyFatalErrors_; }
 
   Prescanner &set_fixedForm(bool yes) {
@@ -43,13 +43,12 @@ public:
     return *this;
   }
 
-  const AllSources &allSources() const { return messages_.allSources(); }
+  AllSources *allSources() const { return allSources_; }
 
-  CookedSource Prescan(AllSources *);
+  CookedSource Prescan();
   std::optional<TokenSequence> NextTokenizedLine();
   Provenance GetCurrentProvenance() const { return GetProvenance(at_); }
-  std::string GetCurrentPath() const;  // __FILE__
-  int GetCurrentLineNumber() const;  // __LINE__
+  Provenance CompilerInsertionProvenance(char ch) const;
 
 private:
   void BeginSourceLine(const char *at) {
@@ -97,7 +96,8 @@ private:
   bool FreeFormContinuation();
   void PayNewlineDebt(CookedSource *);
 
-  Messages &messages_;
+  Messages *messages_;
+  AllSources *allSources_;
 
   Provenance startProvenance_;
   const char *start_{nullptr};  // beginning of sourceFile_ content
@@ -118,6 +118,7 @@ private:
   bool enableBackslashEscapesInCharLiterals_{true};
   int delimiterNesting_{0};
   Preprocessor preprocessor_;
+  std::map<char, Provenance> compilerInsertionProvenance_;
 };
 }  // namespace parser
 }  // namespace Fortran

diff --git a/flang/tools/f18/f18.cc b/flang/tools/f18/f18.cc
index d71ae1a..89bbc5e 100644 (file)
--- a/flang/tools/f18/f18.cc
+++ b/flang/tools/f18/f18.cc
@@ -106,13 +106,13 @@ int main(int argc, char *const argv[]) {
 
   Fortran::parser::AllSources allSources{sourceFile};
   Fortran::parser::Messages messages{allSources};
-  Fortran::parser::Prescanner prescanner{messages};
+  Fortran::parser::Prescanner prescanner{&messages, &allSources};
   Fortran::parser::CookedSource cooked{
       prescanner.set_fixedForm(fixedForm)
           .set_enableBackslashEscapesInCharLiterals(backslashEscapes)
           .set_fixedFormColumnLimit(columns)
           .set_enableOldDebugLines(enableOldDebugLines)
-          .Prescan(&allSources)};
+          .Prescan()};
   messages.Emit(std::cerr);
   if (prescanner.anyFatalErrors()) {
     return 1;