From 97a7c501dc7c0bc360b4dd104a5c8d8963d97d9d Mon Sep 17 00:00:00 2001 From: peter klausler Date: Mon, 12 Feb 2018 14:43:16 -0800 Subject: [PATCH] [flang] Forge provenances for compiler-inserted text. Original-commit: flang-compiler/f18@dbdd01aa258557d45816d1a1e0588553c16bef6a Reviewed-on: https://github.com/flang-compiler/f18/pull/9 Tree-same-pre-rewrite: false --- flang/lib/parser/preprocessor.cc | 58 ++++++++++++++++++++++++---------------- flang/lib/parser/preprocessor.h | 9 ++++--- flang/lib/parser/prescan.cc | 30 +++++++++++++++------ flang/lib/parser/prescan.h | 17 ++++++------ flang/tools/f18/f18.cc | 4 +-- 5 files changed, 73 insertions(+), 45 deletions(-) diff --git a/flang/lib/parser/preprocessor.cc b/flang/lib/parser/preprocessor.cc index 4b703cb..b4fdbf0 100644 --- a/flang/lib/parser/preprocessor.cc +++ b/flang/lib/parser/preprocessor.cc @@ -132,8 +132,9 @@ Definition::Definition(const std::vector &argNames, argumentCount_(argNames.size()), isVariadic_{isVariadic}, replacement_{Tokenize(argNames, repl, firstToken, tokens)} {} -Definition::Definition(const std::string &predefined) - : isPredefined_{true}, replacement_{predefined} {} +Definition::Definition(const std::string &predefined, AllSources *sources) + : isPredefined_{true}, replacement_{predefined, + sources->AddCompilerInsertion(predefined).start} {} bool Definition::set_isDisabled(bool disable) { bool was{isDisabled_}; @@ -172,7 +173,8 @@ TokenSequence Definition::Tokenize(const std::vector &argNames, return result; } -TokenSequence Definition::Apply(const std::vector &args) { +TokenSequence Definition::Apply( + const std::vector &args, const Prescanner &prescanner) { TokenSequence result; bool pasting{false}; bool skipping{false}; @@ -208,7 +210,8 @@ TokenSequence Definition::Apply(const std::vector &args) { while (result.size() >= afterLastNonBlank) { result.pop_back(); } - result.PutNextTokenChar('"', 0); // TODO provenance + result.PutNextTokenChar( + '"', prescanner.CompilerInsertionProvenance('"')); for (size_t k{0}; k < argTokens; ++k) { const CharPointerWithLength &arg{args[index][k]}; size_t argBytes{args[index][k].size()}; @@ -221,7 +224,8 @@ TokenSequence Definition::Apply(const std::vector &args) { result.PutNextTokenChar(ch, from); } } - result.PutNextTokenChar('"', 0); // TODO provenance + result.PutNextTokenChar( + '"', prescanner.CompilerInsertionProvenance('"')); result.CloseToken(); } else { for (size_t k{0}; k < argTokens; ++k) { @@ -247,7 +251,7 @@ TokenSequence Definition::Apply(const std::vector &args) { token.ToString() == "__VA_ARGS__") { for (size_t k{argumentCount_}; k < args.size(); ++k) { if (k > argumentCount_) { - result.Put(","s, 0); // TODO provenance + result.Put(","s, prescanner.CompilerInsertionProvenance(',')); } result.Put(args[k]); } @@ -283,13 +287,15 @@ Preprocessor::Preprocessor(Prescanner &ps) : prescanner_{ps} { // of __DATE__ or __TIME__ change during compilation. std::time_t now; std::time(&now); - definitions_.emplace(SaveToken("__DATE__"s), // e.g., "Jun 16 1904" - Definition{FormatTime(now, "\"%h %e %Y\""), 0, 1}); - definitions_.emplace(SaveToken("__TIME__"s), // e.g., "23:59:60" - Definition{FormatTime(now, "\"%T\""), 0, 1}); + definitions_.emplace(SaveTokenAsName("__DATE__"s), // e.g., "Jun 16 1904" + Definition{FormatTime(now, "\"%h %e %Y\""), ps.allSources()}); + definitions_.emplace(SaveTokenAsName("__TIME__"s), // e.g., "23:59:60" + Definition{FormatTime(now, "\"%T\""), ps.allSources()}); // The values of these predefined macros depend on their invocation sites. - definitions_.emplace(SaveToken("__FILE__"s), Definition{"__FILE__"s}); - definitions_.emplace(SaveToken("__LINE__"s), Definition{"__LINE__"s}); + definitions_.emplace( + SaveTokenAsName("__FILE__"s), Definition{"__FILE__"s, ps.allSources()}); + definitions_.emplace( + SaveTokenAsName("__LINE__"s), Definition{"__LINE__"s, ps.allSources()}); } bool Preprocessor::MacroReplacement( @@ -328,17 +334,21 @@ bool Preprocessor::MacroReplacement( if (def.isPredefined()) { std::string name{def.replacement()[0].ToString()}; if (name == "__FILE__") { - result->Put("\""s + - prescanner_.allSources().GetPath( + std::string f{"\""s + + prescanner_.allSources()->GetPath( prescanner_.GetCurrentProvenance()) + - '"'); + '"'}; + result->Put( + f, prescanner_.allSources()->AddCompilerInsertion(f).start); continue; } if (name == "__LINE__") { std::stringstream ss; - ss << prescanner_.allSources().GetLineNumber( + ss << prescanner_.allSources()->GetLineNumber( prescanner_.GetCurrentProvenance()); - result->Put(ss.str()); + std::string s{ss.str()}; + result->Put( + s, prescanner_.allSources()->AddCompilerInsertion(s).start); continue; } } @@ -391,7 +401,7 @@ bool Preprocessor::MacroReplacement( args.emplace_back(TokenSequence(input, at, count)); } def.set_isDisabled(true); - result->Put(ReplaceMacros(def.Apply(args))); + result->Put(ReplaceMacros(def.Apply(args, prescanner_))); def.set_isDisabled(false); } return true; @@ -478,7 +488,7 @@ bool Preprocessor::Directive(const TokenSequence &dir) { Complain("#define: missing or invalid name"); return false; } - nameToken = SaveToken(nameToken); + nameToken = SaveTokenAsName(nameToken); definitions_.erase(nameToken); if (++j < tokens && dir[j].size() == 1 && dir[j][0] == '(') { j = SkipBlanks(dir, j + 1, tokens); @@ -615,7 +625,8 @@ bool Preprocessor::Directive(const TokenSequence &dir) { return false; } -CharPointerWithLength Preprocessor::SaveToken(const CharPointerWithLength &t) { +CharPointerWithLength Preprocessor::SaveTokenAsName( + const CharPointerWithLength &t) { names_.push_back(t.ToString()); return {names_.back().data(), names_.back().size()}; } @@ -652,7 +663,7 @@ bool Preprocessor::SkipDisabledConditionalCode( } void Preprocessor::Complain(const std::string &message) { - prescanner_.messages().Put({prescanner_.GetCurrentProvenance(), message}); + prescanner_.messages()->Put({prescanner_.GetCurrentProvenance(), message}); } // Precedence level codes used here to accommodate mixed Fortran and C: @@ -661,7 +672,7 @@ void Preprocessor::Complain(const std::string &message) { // 13: ** // 12: *, /, % (modulus) // 11: + and - -// 0: << and >> +// 10: << and >> // 9: bitwise & // 8: bitwise ^ // 7: bitwise | @@ -942,7 +953,8 @@ bool Preprocessor::IsIfPredicateTrue( name = expr1[j++]; } if (!name.empty()) { - expr2.Put(IsNameDefined(name) ? "1" : "0", 1, 0); // TODO provenance + char truth{IsNameDefined(name) ? '1' : '0'}; + expr2.Put(&truth, 1, prescanner_.CompilerInsertionProvenance(truth)); continue; } } diff --git a/flang/lib/parser/preprocessor.h b/flang/lib/parser/preprocessor.h index c799f0d..b30602c 100644 --- a/flang/lib/parser/preprocessor.h +++ b/flang/lib/parser/preprocessor.h @@ -91,7 +91,7 @@ public: : start_{std::move(that.start_)}, nextStart_{that.nextStart_}, char_{std::move(that.char_)}, provenances_{std::move(that.provenances_)} { } - TokenSequence(const std::string &s) { Put(s, 0); } // TODO predefined prov. + TokenSequence(const std::string &s, Provenance p) { Put(s, p); } TokenSequence &operator=(const TokenSequence &that) { clear(); @@ -159,7 +159,7 @@ public: Definition(const TokenSequence &, size_t firstToken, size_t tokens); Definition(const std::vector &argNames, const TokenSequence &, size_t firstToken, size_t tokens, bool isVariadic = false); - explicit Definition(const std::string &predefined); + Definition(const std::string &predefined, AllSources *); bool isFunctionLike() const { return isFunctionLike_; } size_t argumentCount() const { return argumentCount_; } @@ -170,7 +170,8 @@ public: bool set_isDisabled(bool disable); - TokenSequence Apply(const std::vector &args); + TokenSequence Apply( + const std::vector &args, const Prescanner &); private: static TokenSequence Tokenize(const std::vector &argNames, @@ -203,7 +204,7 @@ private: enum class CanDeadElseAppear { No, Yes }; void Complain(const std::string &); - CharPointerWithLength SaveToken(const CharPointerWithLength &); + CharPointerWithLength SaveTokenAsName(const CharPointerWithLength &); bool IsNameDefined(const CharPointerWithLength &); TokenSequence ReplaceMacros(const TokenSequence &); bool SkipDisabledConditionalCode(const std::string &dirName, IsElseActive); diff --git a/flang/lib/parser/prescan.cc b/flang/lib/parser/prescan.cc index 518471a..139ddf2 100644 --- a/flang/lib/parser/prescan.cc +++ b/flang/lib/parser/prescan.cc @@ -9,14 +9,23 @@ namespace Fortran { namespace parser { -CookedSource Prescanner::Prescan(AllSources *allSources) { +Prescanner::Prescanner(Messages *messages, AllSources *allSources) + : messages_{messages}, allSources_{allSources}, preprocessor_{*this} { + std::string compilerInserts{" ,\"01"}; + ProvenanceRange range{allSources->AddCompilerInsertion(compilerInserts)}; + for (size_t j{0}; j < compilerInserts.size(); ++j) { + compilerInsertionProvenance_[compilerInserts[j]] = range.start + j; + } +} + +CookedSource Prescanner::Prescan() { startProvenance_ = 0; - start_ = &(*allSources)[0]; - limit_ = start_ + allSources->size(); + start_ = &(*allSources_)[0]; + limit_ = start_ + allSources_->size(); lineStart_ = start_; BeginSourceLine(start_); TokenSequence tokens, preprocessed; - CookedSource cooked{allSources}; + CookedSource cooked{allSources_}; while (lineStart_ < limit_) { if (CommentLinesAndPreprocessorDirectives() && lineStart_ >= limit_) { break; @@ -68,6 +77,10 @@ std::optional Prescanner::NextTokenizedLine() { return {std::move(tokens)}; } +Provenance Prescanner::CompilerInsertionProvenance(char ch) const { + return compilerInsertionProvenance_.find(ch)->second; +} + void Prescanner::NextLine() { void *vstart{static_cast(const_cast(lineStart_))}; void *v{std::memchr(vstart, '\n', limit_ - lineStart_)}; @@ -98,8 +111,9 @@ void Prescanner::LabelField(TokenSequence *token) { token->CloseToken(); } if (outCol < 7) { + Provenance provenance{CompilerInsertionProvenance(' ')}; for (; outCol < 7; ++outCol) { - EmitChar(token, ' '); + token->PutNextTokenChar(' ', provenance); } token->CloseToken(); } @@ -192,7 +206,7 @@ bool Prescanner::NextToken(TokenSequence *tokens) { inCharLiteral_ = true; while (n-- > 0) { if (PadOutCharacterLiteral()) { - EmitChar(tokens, ' '); + tokens->PutNextTokenChar(' ', compilerInsertionProvenance_[' ']); } else { if (*at_ == '\n') { break; // TODO error @@ -283,12 +297,12 @@ void Prescanner::QuotedCharacterLiteral(TokenSequence *tokens) { do { EmitCharAndAdvance(tokens, *at_); while (PadOutCharacterLiteral()) { - EmitChar(tokens, ' '); + tokens->PutNextTokenChar(' ', compilerInsertionProvenance_[' ']); } if (*at_ == '\\' && enableBackslashEscapesInCharLiterals_) { EmitCharAndAdvance(tokens, '\\'); while (PadOutCharacterLiteral()) { - EmitChar(tokens, ' '); + tokens->PutNextTokenChar(' ', compilerInsertionProvenance_[' ']); } } else if (*at_ == quote) { // A doubled quote mark becomes a single instance of the quote character diff --git a/flang/lib/parser/prescan.h b/flang/lib/parser/prescan.h index 819e394..a4c22b3 100644 --- a/flang/lib/parser/prescan.h +++ b/flang/lib/parser/prescan.h @@ -12,6 +12,7 @@ #include "preprocessor.h" #include "provenance.h" #include "source.h" +#include #include #include @@ -20,10 +21,9 @@ namespace parser { class Prescanner { public: - explicit Prescanner(Messages &messages) - : messages_{messages}, preprocessor_{*this} {} + Prescanner(Messages *, AllSources *); - Messages &messages() const { return messages_; } + Messages *messages() const { return messages_; } bool anyFatalErrors() const { return anyFatalErrors_; } Prescanner &set_fixedForm(bool yes) { @@ -43,13 +43,12 @@ public: return *this; } - const AllSources &allSources() const { return messages_.allSources(); } + AllSources *allSources() const { return allSources_; } - CookedSource Prescan(AllSources *); + CookedSource Prescan(); std::optional NextTokenizedLine(); Provenance GetCurrentProvenance() const { return GetProvenance(at_); } - std::string GetCurrentPath() const; // __FILE__ - int GetCurrentLineNumber() const; // __LINE__ + Provenance CompilerInsertionProvenance(char ch) const; private: void BeginSourceLine(const char *at) { @@ -97,7 +96,8 @@ private: bool FreeFormContinuation(); void PayNewlineDebt(CookedSource *); - Messages &messages_; + Messages *messages_; + AllSources *allSources_; Provenance startProvenance_; const char *start_{nullptr}; // beginning of sourceFile_ content @@ -118,6 +118,7 @@ private: bool enableBackslashEscapesInCharLiterals_{true}; int delimiterNesting_{0}; Preprocessor preprocessor_; + std::map compilerInsertionProvenance_; }; } // namespace parser } // namespace Fortran diff --git a/flang/tools/f18/f18.cc b/flang/tools/f18/f18.cc index d71ae1a..89bbc5e 100644 --- a/flang/tools/f18/f18.cc +++ b/flang/tools/f18/f18.cc @@ -106,13 +106,13 @@ int main(int argc, char *const argv[]) { Fortran::parser::AllSources allSources{sourceFile}; Fortran::parser::Messages messages{allSources}; - Fortran::parser::Prescanner prescanner{messages}; + Fortran::parser::Prescanner prescanner{&messages, &allSources}; Fortran::parser::CookedSource cooked{ prescanner.set_fixedForm(fixedForm) .set_enableBackslashEscapesInCharLiterals(backslashEscapes) .set_fixedFormColumnLimit(columns) .set_enableOldDebugLines(enableOldDebugLines) - .Prescan(&allSources)}; + .Prescan()}; messages.Emit(std::cerr); if (prescanner.anyFatalErrors()) { return 1; -- 2.7.4