[flang] Complete provenance tracking through macro calls.

Original-commit: flang-compiler/f18@8c05a6543b647299104ae07a6a248b67f13ee24e
Reviewed-on: https://github.com/flang-compiler/f18/pull/9
Tree-same-pre-rewrite: false

diff --git a/flang/lib/parser/basic-parsers.h b/flang/lib/parser/basic-parsers.h
index 076aef8..8b655a1 100644 (file)
--- a/flang/lib/parser/basic-parsers.h
+++ b/flang/lib/parser/basic-parsers.h
@@ -1204,7 +1204,7 @@ public:
   constexpr WithinCharLiteral(const WithinCharLiteral &) = default;
   constexpr WithinCharLiteral(const PA &parser) : parser_{parser} {}
   std::optional<resultType> Parse(ParseState *state) const {
-    bool was = state->set_inCharLiteral(true);
+    bool was{state->inCharLiteral()};
     std::optional<resultType> result{parser_.Parse(state)};
     state->set_inCharLiteral(was);
     return result;

diff --git a/flang/lib/parser/cooked-tokens.h b/flang/lib/parser/cooked-tokens.h
index 030e08a..b358a15 100644 (file)
--- a/flang/lib/parser/cooked-tokens.h
+++ b/flang/lib/parser/cooked-tokens.h
@@ -249,7 +249,7 @@ template<char quote> struct CharLiteral {
   using resultType = std::string;
   static std::optional<std::string> Parse(ParseState *state) {
     std::string str;
-    CHECK(!state->set_inCharLiteral(true));
+    CHECK(!state->inCharLiteral());
     static constexpr auto nextch = attempt(CharLiteralChar{});
     while (std::optional<CharLiteralChar::Result> ch{nextch.Parse(state)}) {
       if (ch->ch == quote && !ch->wasEscaped) {
@@ -395,7 +395,8 @@ struct HollerithLiteral {
       return {};
     }
     std::string content;
-    CHECK(!state->set_inCharLiteral(true));
+    CHECK(!state->inCharLiteral());
+    state->set_inCharLiteral(true);
     for (auto j = *charCount; j-- > 0;) {
       std::optional<char> ch{cookedNextChar.Parse(state)};
       if (!ch || !isprint(*ch)) {

diff --git a/flang/lib/parser/message.cc b/flang/lib/parser/message.cc
index c8a2aa0..9ee48de 100644 (file)
--- a/flang/lib/parser/message.cc
+++ b/flang/lib/parser/message.cc
@@ -3,16 +3,20 @@
 namespace Fortran {
 namespace parser {
 
-void Message::Emit(std::ostream &o, const AllSources &sources) const {
-  if (context_) {
-    context_->Emit(o, sources);
+Provenance Message::Emit(
+    std::ostream &o, const AllSources &sources, bool echoSourceLine) const {
+  if (!context_ || context_->Emit(o, sources, false) != provenance_) {
+    sources.Identify(o, provenance_, "", echoSourceLine);
   }
-  sources.Identify(o, provenance_, "");
   o << "   " << message_ << '\n';
+  return provenance_;
 }
 
 void Messages::Emit(std::ostream &o) const {
   for (const auto &msg : messages_) {
+    if (msg.context()) {
+      o << "In the context ";
+    }
     msg.Emit(o, allSources_);
   }
 }

diff --git a/flang/lib/parser/message.h b/flang/lib/parser/message.h
index b3a3f89..f271899 100644 (file)
--- a/flang/lib/parser/message.h
+++ b/flang/lib/parser/message.h
@@ -42,7 +42,8 @@ public:
   std::string message() const { return message_; }
   MessageContext context() const { return context_; }
 
-  void Emit(std::ostream &, const AllSources &) const;
+  Provenance Emit(
+      std::ostream &, const AllSources &, bool echoSourceLine = true) const;
 
 private:
   Provenance provenance_;

diff --git a/flang/lib/parser/preprocessor.cc b/flang/lib/parser/preprocessor.cc
index c821555..d9185ae 100644 (file)
--- a/flang/lib/parser/preprocessor.cc
+++ b/flang/lib/parser/preprocessor.cc
@@ -58,7 +58,7 @@ TokenSequence Definition::Tokenize(const std::vector<std::string> &argNames,
     if (IsIdentifierFirstCharacter(tok)) {
       auto it = args.find(tok.ToString());
       if (it != args.end()) {
-        result.Put(it->second, token.GetProvenance(j));
+        result.Put(it->second, token.GetTokenProvenance(j));
         continue;
       }
     }
@@ -111,7 +111,7 @@ TokenSequence Definition::Apply(
           size_t argBytes{args[index][k].size()};
           for (size_t n{0}; n < argBytes; ++n) {
             char ch{arg[n]};
-            Provenance from{args[index].GetProvenance(k, n)};
+            Provenance from{args[index].GetTokenProvenance(k, n)};
             if (ch == '"' || ch == '\\') {
               result.PutNextTokenChar(ch, from);
             }
@@ -238,14 +238,19 @@ bool Preprocessor::MacroReplacement(const TokenSequence &input,
         if (!repl.empty()) {
           ProvenanceRange insert{allSources_->AddCompilerInsertion(repl)};
           ProvenanceRange call{allSources_->AddMacroCall(
-              insert, input.GetProvenanceRange(j), repl)};
+              insert, input.GetTokenProvenanceRange(j), repl)};
           result->Put(repl, call.LocalOffsetToProvenance(0));
           continue;
         }
       }
       def.set_isDisabled(true);
-      result->Put(ReplaceMacros(def.replacement(), prescanner));
+      TokenSequence replaced{ReplaceMacros(def.replacement(), prescanner)};
       def.set_isDisabled(false);
+      ProvenanceRange from{def.replacement().GetProvenanceRange()};
+      ProvenanceRange use{input.GetTokenProvenanceRange(j)};
+      ProvenanceRange newRange{
+          allSources_->AddMacroCall(from, use, replaced.ToString())};
+      result->Put(replaced, newRange);
       continue;
     }
     // Possible function-like macro call.  Skip spaces and newlines to see
@@ -284,16 +289,22 @@ bool Preprocessor::MacroReplacement(const TokenSequence &input,
       result->Put(input, j);
       continue;
     }
-    j = k;  // advance to the terminal ')'
     std::vector<TokenSequence> args;
-    for (k = 0; k < argStart.size(); ++k) {
-      size_t at{argStart[k]};
-      size_t count{(k + 1 == argStart.size() ? j : argStart[k + 1] - 1) - at};
+    for (size_t n{0}; n < argStart.size(); ++n) {
+      size_t at{argStart[n]};
+      size_t count{(n + 1 == argStart.size() ? k : argStart[n + 1] - 1) - at};
       args.emplace_back(TokenSequence(input, at, count));
     }
     def.set_isDisabled(true);
-    result->Put(ReplaceMacros(def.Apply(args, *allSources_), prescanner));
+    TokenSequence replaced{
+        ReplaceMacros(def.Apply(args, *allSources_), prescanner)};
     def.set_isDisabled(false);
+    ProvenanceRange from{def.replacement().GetProvenanceRange()};
+    ProvenanceRange use{input.GetIntervalProvenanceRange(j, k - j)};
+    ProvenanceRange newRange{
+        allSources_->AddMacroCall(from, use, replaced.ToString())};
+    result->Put(replaced, newRange);
+    j = k;  // advance to the terminal ')'
   }
   return true;
 }
@@ -521,7 +532,6 @@ bool Preprocessor::Directive(const TokenSequence &dir, Prescanner *prescanner) {
       prescanner->Complain("#include: missing name of file to include");
       return false;
     }
-    ProvenanceRange includeDirRange{dir.GetProvenanceRange(j)};
     std::string include;
     if (dir[j].ToString() == "<") {
       if (dir[tokens - 1].ToString() != ">") {
@@ -549,7 +559,7 @@ bool Preprocessor::Directive(const TokenSequence &dir, Prescanner *prescanner) {
       return false;
     }
     ProvenanceRange fileRange{
-        allSources_->AddIncludedFile(*included, includeDirRange)};
+        allSources_->AddIncludedFile(*included, dir.GetProvenanceRange())};
     return Prescanner{*prescanner}.Prescan(fileRange);
   }
   prescanner->Complain("#"s + dirName + ": unknown or unimplemented directive");

diff --git a/flang/lib/parser/provenance.cc b/flang/lib/parser/provenance.cc
index 40b1bd4..bdd67c1 100644 (file)
--- a/flang/lib/parser/provenance.cc
+++ b/flang/lib/parser/provenance.cc
@@ -125,8 +125,8 @@ ProvenanceRange AllSources::AddCompilerInsertion(std::string text) {
   return covers;
 }
 
-void AllSources::Identify(
-    std::ostream &o, Provenance at, const std::string &prefix) const {
+void AllSources::Identify(std::ostream &o, Provenance at,
+    const std::string &prefix, bool echoSourceLine) const {
   CHECK(IsValid(at));
   static const std::string indented{prefix + "  "};
   const Origin &origin{MapToOrigin(at)};
@@ -135,11 +135,28 @@ void AllSources::Identify(
           [&](const Inclusion &inc) {
             size_t offset{origin.covers.ProvenanceToLocalOffset(at)};
             std::pair<int, int> pos{inc.source.FindOffsetLineAndColumn(offset)};
-            o << prefix << "at line " << pos.first << ", column " << pos.second
-              << " in the " << (inc.isModule ? "module " : "file ")
+            o << prefix << "at line " << pos.first << ", column " << pos.second;
+            if (echoSourceLine) {
+              o << ":\n" << indented << "  ";
+              const char *text{inc.source.content() +
+                  inc.source.GetLineStartOffset(pos.first)};
+              for (const char *p{text}; *p != '\n'; ++p) {
+                o << *p;
+              }
+              o << '\n' << indented << "  ";
+              for (int j{1}; j < pos.second; ++j) {
+                char ch{text[j - 1]};
+                o << (ch == '\t' ? '\t' : ' ');
+              }
+              o << "^\n" << prefix;
+            } else {
+              o << ' ';
+            }
+            o << "in the " << (inc.isModule ? "module " : "file ")
               << inc.source.path() << '\n';
             if (IsValid(origin.replaces)) {
-              o << prefix << " that was " << (inc.isModule ? "used\n" : "included\n");
+              o << prefix << " that was "
+                << (inc.isModule ? "used\n" : "included\n");
               Identify(o, origin.replaces.LocalOffsetToProvenance(0), indented);
             }
           },
@@ -206,9 +223,8 @@ Provenance AllSources::CompilerInsertionProvenance(char ch) const {
 
 AllSources::Origin::Origin(ProvenanceRange r, const SourceFile &source)
   : u{Inclusion{source}}, covers{r} {}
-AllSources::Origin::Origin(
-    ProvenanceRange r, const SourceFile &included, ProvenanceRange from,
-    bool isModule)
+AllSources::Origin::Origin(ProvenanceRange r, const SourceFile &included,
+    ProvenanceRange from, bool isModule)
   : u{Inclusion{included, isModule}}, covers{r}, replaces{from} {}
 AllSources::Origin::Origin(ProvenanceRange r, ProvenanceRange def,
     ProvenanceRange use, const std::string &expansion)
@@ -284,7 +300,9 @@ void AllSources::Dump(std::ostream &o) const {
     m.covers.Dump(o);
     o << " -> ";
     std::visit(visitors{[&](const Inclusion &inc) {
-                          if (inc.isModule) { o << "module "; }
+                          if (inc.isModule) {
+                            o << "module ";
+                          }
                           o << "file " << inc.source.path();
                         },
                    [&](const Macro &mac) { o << "macro " << mac.expansion; },

diff --git a/flang/lib/parser/provenance.h b/flang/lib/parser/provenance.h
index c674326..4949d00 100644 (file)
--- a/flang/lib/parser/provenance.h
+++ b/flang/lib/parser/provenance.h
@@ -48,8 +48,9 @@ public:
   }
   Provenance operator+(size_t n) const { return {offset_ + n}; }
   bool operator<(Provenance that) const { return offset_ < that.offset_; }
-  bool operator<=(Provenance that) const { return offset_ <= that.offset_; }
+  bool operator<=(Provenance that) const { return !(that < *this); }
   bool operator==(Provenance that) const { return offset_ == that.offset_; }
+  bool operator!=(Provenance that) const { return !(*this == that); }
   size_t offset() const { return offset_; }
 
 private:
@@ -159,8 +160,8 @@ public:
   std::string PopSearchPathDirectory();
   const SourceFile *Open(std::string path, std::stringstream *error);
 
-  ProvenanceRange AddIncludedFile(const SourceFile &, ProvenanceRange,
-                                  bool isModule = false);
+  ProvenanceRange AddIncludedFile(
+      const SourceFile &, ProvenanceRange, bool isModule = false);
   ProvenanceRange AddMacroCall(
       ProvenanceRange def, ProvenanceRange use, const std::string &expansion);
   ProvenanceRange AddCompilerInsertion(std::string);
@@ -169,7 +170,8 @@ public:
   bool IsValid(ProvenanceRange range) const {
     return range.size() > 0 && range_.Contains(range);
   }
-  void Identify(std::ostream &, Provenance, const std::string &prefix) const;
+  void Identify(std::ostream &, Provenance, const std::string &prefix,
+      bool echoSourceLine = false) const;
   const SourceFile *GetSourceFile(Provenance, size_t *offset = nullptr) const;
   ProvenanceRange GetContiguousRangeAround(ProvenanceRange) const;
   std::string GetPath(Provenance) const;  // __FILE__
@@ -196,7 +198,7 @@ private:
   struct Origin {
     Origin(ProvenanceRange, const SourceFile &);
     Origin(ProvenanceRange, const SourceFile &, ProvenanceRange,
-           bool isModule = false);
+        bool isModule = false);
     Origin(ProvenanceRange, ProvenanceRange def, ProvenanceRange use,
         const std::string &expansion);
     Origin(ProvenanceRange, const std::string &);

diff --git a/flang/lib/parser/source.h b/flang/lib/parser/source.h
index 95e8521..d28837d 100644 (file)
--- a/flang/lib/parser/source.h
+++ b/flang/lib/parser/source.h
@@ -29,6 +29,9 @@ public:
   bool Open(std::string path, std::stringstream *error);
   void Close();
   std::pair<int, int> FindOffsetLineAndColumn(size_t) const;
+  size_t GetLineStartOffset(int lineNumber) const {
+    return lineStart_.at(lineNumber - 1);
+  }
 
 private:
   std::string path_;

diff --git a/flang/lib/parser/token-sequence.cc b/flang/lib/parser/token-sequence.cc
index 99f7cd0..5c2a2f9 100644 (file)
--- a/flang/lib/parser/token-sequence.cc
+++ b/flang/lib/parser/token-sequence.cc
@@ -47,6 +47,16 @@ void TokenSequence::Put(const TokenSequence &that) {
   provenances_.Put(that.provenances_);
 }
 
+void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
+  size_t offset{0};
+  for (size_t j{0}; j < that.size(); ++j) {
+    CharPointerWithLength tok{that[j]};
+    Put(tok, range.LocalOffsetToProvenance(offset));
+    offset += tok.size();
+  }
+  CHECK(offset == range.size());
+}
+
 void TokenSequence::Put(const TokenSequence &that, size_t at, size_t tokens) {
   ProvenanceRange provenance;
   size_t offset{0};
@@ -74,6 +84,7 @@ void TokenSequence::Put(const char *s, size_t bytes, Provenance provenance) {
 void TokenSequence::Put(const CharPointerWithLength &t, Provenance provenance) {
   Put(&t[0], t.size(), provenance);
 }
+
 void TokenSequence::Put(const std::string &s, Provenance provenance) {
   Put(s.data(), s.size(), provenance);
 }
@@ -104,15 +115,32 @@ std::string TokenSequence::ToString() const {
   return {&char_[0], char_.size()};
 }
 
-Provenance TokenSequence::GetProvenance(size_t token, size_t offset) const {
+Provenance TokenSequence::GetTokenProvenance(
+    size_t token, size_t offset) const {
   ProvenanceRange range{provenances_.Map(start_[token] + offset)};
   return range.LocalOffsetToProvenance(0);
 }
 
-ProvenanceRange TokenSequence::GetProvenanceRange(
+ProvenanceRange TokenSequence::GetTokenProvenanceRange(
     size_t token, size_t offset) const {
   ProvenanceRange range{provenances_.Map(start_[token] + offset)};
   return range.Prefix(TokenBytes(token) - offset);
 }
+
+ProvenanceRange TokenSequence::GetIntervalProvenanceRange(
+    size_t token, size_t tokens) const {
+  if (tokens == 0) {
+    return {};
+  }
+  ProvenanceRange range{provenances_.Map(start_[token])};
+  while (--tokens > 0 &&
+      range.AnnexIfPredecessor(provenances_.Map(start_[++token]))) {
+  }
+  return range;
+}
+
+ProvenanceRange TokenSequence::GetProvenanceRange() const {
+  return GetIntervalProvenanceRange(0, start_.size());
+}
 }  // namespace parser
 }  // namespace Fortran

diff --git a/flang/lib/parser/token-sequence.h b/flang/lib/parser/token-sequence.h
index 324ac0f..b3e71ea 100644 (file)
--- a/flang/lib/parser/token-sequence.h
+++ b/flang/lib/parser/token-sequence.h
@@ -121,6 +121,7 @@ public:
   }
 
   void Put(const TokenSequence &);
+  void Put(const TokenSequence &, ProvenanceRange);
   void Put(const TokenSequence &, size_t at, size_t tokens = 1);
   void Put(const char *, size_t, Provenance);
   void Put(const CharPointerWithLength &, Provenance);
@@ -128,8 +129,12 @@ public:
   void Put(const std::stringstream &, Provenance);
   void EmitWithCaseConversion(CookedSource *) const;
   std::string ToString() const;
-  Provenance GetProvenance(size_t token, size_t offset = 0) const;
-  ProvenanceRange GetProvenanceRange(size_t token, size_t offset = 0) const;
+  Provenance GetTokenProvenance(size_t token, size_t offset = 0) const;
+  ProvenanceRange GetTokenProvenanceRange(
+      size_t token, size_t offset = 0) const;
+  ProvenanceRange GetIntervalProvenanceRange(
+      size_t token, size_t tokens = 1) const;
+  ProvenanceRange GetProvenanceRange() const;
 
 private:
   size_t TokenBytes(size_t token) const {

diff --git a/flang/tools/f18/f18.cc b/flang/tools/f18/f18.cc
index 8045b70..d7c27fd 100644 (file)
--- a/flang/tools/f18/f18.cc
+++ b/flang/tools/f18/f18.cc
@@ -138,11 +138,11 @@ int main(int argc, char *const argv[]) {
   Fortran::parser::ParseState state{cooked};
   Fortran::parser::UserState ustate;
   state.set_inFixedForm(fixedForm)
-       .set_enableBackslashEscapesInCharLiterals(backslashEscapes)
-       .set_strictConformance(standard)
-       .set_columns(columns)
-       .set_enableOldDebugLines(enableOldDebugLines)
-       .set_userState(&ustate);
+      .set_enableBackslashEscapesInCharLiterals(backslashEscapes)
+      .set_strictConformance(standard)
+      .set_columns(columns)
+      .set_enableOldDebugLines(enableOldDebugLines)
+      .set_userState(&ustate);
 
   if (dumpCookedChars) {
     while (std::optional<char> och{