From 92a541978618674ce112b2f500853218fed24db8 Mon Sep 17 00:00:00 2001 From: peter klausler Date: Mon, 31 Aug 2020 12:22:24 -0700 Subject: [PATCH] [flang] Support multiple CookedSource instances These are owned by an instance of a new class AllCookedSources. This removes the need for a Scope to own a string containing a module's cooked source stream, and will enable errors to be emitted when parsing module files in the future. Differential Revision: https://reviews.llvm.org/D86891 --- flang/include/flang/Lower/Bridge.h | 12 +-- flang/include/flang/Lower/ConvertType.h | 5 -- flang/include/flang/Parser/instrumented-parser.h | 2 +- flang/include/flang/Parser/message.h | 16 ++-- flang/include/flang/Parser/parsing.h | 13 +-- flang/include/flang/Parser/provenance.h | 61 +++++++++---- flang/include/flang/Parser/user-state.h | 11 +-- flang/include/flang/Semantics/scope.h | 5 -- flang/include/flang/Semantics/semantics.h | 13 ++- flang/lib/Parser/debug-parser.cpp | 4 +- flang/lib/Parser/instrumented-parser.cpp | 7 +- flang/lib/Parser/message.cpp | 28 +++--- flang/lib/Parser/parsing.cpp | 28 +++--- flang/lib/Parser/prescan.cpp | 29 +++--- flang/lib/Parser/prescan.h | 14 ++- flang/lib/Parser/provenance.cpp | 107 +++++++++++++++-------- flang/lib/Semantics/mod-file.cpp | 3 +- flang/lib/Semantics/scope.cpp | 8 -- flang/lib/Semantics/semantics.cpp | 9 +- flang/test/Semantics/getsymbols02.f90 | 4 +- flang/tools/f18-parse-demo/f18-parse-demo.cpp | 7 +- flang/tools/f18/f18.cpp | 15 ++-- flang/unittests/Evaluate/intrinsics.cpp | 9 +- 23 files changed, 230 insertions(+), 180 deletions(-) diff --git a/flang/include/flang/Lower/Bridge.h b/flang/include/flang/Lower/Bridge.h index aee7a0e..ebaffaa 100644 --- a/flang/include/flang/Lower/Bridge.h +++ b/flang/include/flang/Lower/Bridge.h @@ -34,7 +34,7 @@ namespace evaluate { class IntrinsicProcTable; } // namespace evaluate namespace parser { -class CookedSource; +class AllCookedSources; struct Program; } // namespace parser namespace semantics { @@ -55,8 +55,8 @@ public: static LoweringBridge create(const Fortran::common::IntrinsicTypeDefaultKinds &defaultKinds, const Fortran::evaluate::IntrinsicProcTable &intrinsics, - const Fortran::parser::CookedSource &cooked) { - return LoweringBridge{defaultKinds, intrinsics, cooked}; + const Fortran::parser::AllCookedSources &allCooked) { + return LoweringBridge{defaultKinds, intrinsics, allCooked}; } //===--------------------------------------------------------------------===// @@ -71,7 +71,7 @@ public: const Fortran::evaluate::IntrinsicProcTable &getIntrinsicTable() const { return intrinsics; } - const Fortran::parser::CookedSource *getCookedSource() const { + const Fortran::parser::AllCookedSources *getCookedSource() const { return cooked; } @@ -99,13 +99,13 @@ private: explicit LoweringBridge( const Fortran::common::IntrinsicTypeDefaultKinds &defaultKinds, const Fortran::evaluate::IntrinsicProcTable &intrinsics, - const Fortran::parser::CookedSource &cooked); + const Fortran::parser::AllCookedSources &); LoweringBridge() = delete; LoweringBridge(const LoweringBridge &) = delete; const Fortran::common::IntrinsicTypeDefaultKinds &defaultKinds; const Fortran::evaluate::IntrinsicProcTable &intrinsics; - const Fortran::parser::CookedSource *cooked; + const Fortran::parser::AllCookedSources *cooked; std::unique_ptr context; std::unique_ptr module; fir::KindMapping kindMap; diff --git a/flang/include/flang/Lower/ConvertType.h b/flang/include/flang/Lower/ConvertType.h index f4046ef..b807d62 100644 --- a/flang/include/flang/Lower/ConvertType.h +++ b/flang/include/flang/Lower/ConvertType.h @@ -48,11 +48,6 @@ template class Type; } // namespace evaluate -namespace parser { -class CharBlock; -class CookedSource; -} // namespace parser - namespace semantics { class Symbol; } // namespace semantics diff --git a/flang/include/flang/Parser/instrumented-parser.h b/flang/include/flang/Parser/instrumented-parser.h index 51dbd5f..1bc1c52 100644 --- a/flang/include/flang/Parser/instrumented-parser.h +++ b/flang/include/flang/Parser/instrumented-parser.h @@ -31,7 +31,7 @@ public: bool Fails(const char *at, const MessageFixedText &tag, ParseState &); void Note(const char *at, const MessageFixedText &tag, bool pass, const ParseState &); - void Dump(llvm::raw_ostream &, const CookedSource &) const; + void Dump(llvm::raw_ostream &, const AllCookedSources &) const; private: struct LogForPosition { diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h index 46a72e0..cd1df0a 100644 --- a/flang/include/flang/Parser/message.h +++ b/flang/include/flang/Parser/message.h @@ -186,14 +186,14 @@ public: bool SortBefore(const Message &that) const; bool IsFatal() const; std::string ToString() const; - std::optional GetProvenanceRange(const CookedSource &) const; - void Emit(llvm::raw_ostream &, const CookedSource &, + std::optional GetProvenanceRange( + const AllCookedSources &) const; + void Emit(llvm::raw_ostream &, const AllCookedSources &, bool echoSourceLine = true) const; - // If this Message or any of its attachments locates itself via a CharBlock - // within a particular CookedSource, replace its location with the - // corresponding ProvenanceRange. - void ResolveProvenances(const CookedSource &); + // If this Message or any of its attachments locates itself via a CharBlock, + // replace its location with the corresponding ProvenanceRange. + void ResolveProvenances(const AllCookedSources &); bool IsMergeable() const { return std::holds_alternative(text_); @@ -255,8 +255,8 @@ public: bool Merge(const Message &); void Merge(Messages &&); void Copy(const Messages &); - void ResolveProvenances(const CookedSource &); - void Emit(llvm::raw_ostream &, const CookedSource &cooked, + void ResolveProvenances(const AllCookedSources &); + void Emit(llvm::raw_ostream &, const AllCookedSources &, bool echoSourceLines = true) const; void AttachTo(Message &); bool AnyFatalError() const; diff --git a/flang/include/flang/Parser/parsing.h b/flang/include/flang/Parser/parsing.h index 9f8bff9..6594f97 100644 --- a/flang/include/flang/Parser/parsing.h +++ b/flang/include/flang/Parser/parsing.h @@ -40,15 +40,17 @@ struct Options { class Parsing { public: - explicit Parsing(AllSources &); + explicit Parsing(AllCookedSources &); ~Parsing(); bool consumedWholeFile() const { return consumedWholeFile_; } const char *finalRestingPlace() const { return finalRestingPlace_; } - CookedSource &cooked() { return cooked_; } + AllCookedSources &allCooked() { return allCooked_; } Messages &messages() { return messages_; } std::optional &parseTree() { return parseTree_; } + const CookedSource &cooked() const { return DEREF(currentCooked_); } + const SourceFile *Prescan(const std::string &path, Options); void DumpCookedChars(llvm::raw_ostream &) const; void DumpProvenance(llvm::raw_ostream &) const; @@ -58,13 +60,14 @@ public: void EmitMessage(llvm::raw_ostream &o, const char *at, const std::string &message, bool echoSourceLine = false) const { - cooked_.allSources().EmitMessage( - o, cooked_.GetProvenanceRange(CharBlock(at)), message, echoSourceLine); + allCooked_.allSources().EmitMessage(o, + allCooked_.GetProvenanceRange(CharBlock(at)), message, echoSourceLine); } private: Options options_; - CookedSource cooked_; + AllCookedSources &allCooked_; + CookedSource *currentCooked_{nullptr}; Messages messages_; bool consumedWholeFile_{false}; const char *finalRestingPlace_{nullptr}; diff --git a/flang/include/flang/Parser/provenance.h b/flang/include/flang/Parser/provenance.h index b543cd7..52aac93 100644 --- a/flang/include/flang/Parser/provenance.h +++ b/flang/include/flang/Parser/provenance.h @@ -17,6 +17,7 @@ #include "flang/Common/interval.h" #include "llvm/Support/raw_ostream.h" #include +#include #include #include #include @@ -213,28 +214,22 @@ private: Encoding encoding_{Encoding::UTF_8}; }; +// Represents the result of preprocessing and prescanning a single source +// file (and all its inclusions) or module file. Parsers operate within +// single instances of CookedSource. class CookedSource { public: - explicit CookedSource(AllSources &); - ~CookedSource(); - - AllSources &allSources() { return allSources_; } - const AllSources &allSources() const { return allSources_; } const std::string &data() const { return data_; } - bool IsValid(const char *p) const { + bool Contains(const char *p) const { return p >= &data_.front() && p <= &data_.back() + 1; } - bool IsValid(CharBlock range) const { - return !range.empty() && IsValid(range.begin()) && IsValid(range.end() - 1); + bool Contains(CharBlock range) const { + return !range.empty() && Contains(range.begin()) && + Contains(range.end() - 1); } - bool IsValid(ProvenanceRange r) const { return allSources_.IsValid(r); } std::optional GetProvenanceRange(CharBlock) const; - std::optional GetCharBlockFromLineAndColumns( - int line, int startColumn, int endColumn) const; - std::optional> - GetSourcePositionRange(CharBlock) const; std::optional GetCharBlock(ProvenanceRange) const; // The result of a Put() is the offset that the new data @@ -256,17 +251,51 @@ public: } std::size_t BufferedBytes() const; - void Marshal(); // marshals text into one contiguous block - void CompileProvenanceRangeToOffsetMappings(); + void Marshal(AllSources &); // marshals text into one contiguous block + void CompileProvenanceRangeToOffsetMappings(AllSources &); std::string AcquireData() { return std::move(data_); } llvm::raw_ostream &Dump(llvm::raw_ostream &) const; private: - AllSources &allSources_; CharBuffer buffer_; // before Marshal() std::string data_; // all of it, prescanned and preprocessed OffsetToProvenanceMappings provenanceMap_; ProvenanceRangeToOffsetMappings invertedMap_; }; + +class AllCookedSources { +public: + explicit AllCookedSources(AllSources &); + ~AllCookedSources(); + + AllSources &allSources() { return allSources_; } + const AllSources &allSources() const { return allSources_; } + + CookedSource &NewCookedSource(); + + template // const char * or CharBlock + const CookedSource *Find(A x) const { + for (const auto &c : cooked_) { + if (c.Contains(x)) { + return &c; + } + } + return nullptr; + } + + bool IsValid(ProvenanceRange r) const { return allSources_.IsValid(r); } + + std::optional GetProvenanceRange(CharBlock) const; + std::optional GetCharBlockFromLineAndColumns( + int line, int startColumn, int endColumn) const; + std::optional> + GetSourcePositionRange(CharBlock) const; + std::optional GetCharBlock(ProvenanceRange) const; + void Dump(llvm::raw_ostream &) const; + +private: + AllSources &allSources_; + std::list cooked_; // owns all CookedSource instances +}; } // namespace Fortran::parser #endif // FORTRAN_PARSER_PROVENANCE_H_ diff --git a/flang/include/flang/Parser/user-state.h b/flang/include/flang/Parser/user-state.h index 75757d2..6a4cf97 100644 --- a/flang/include/flang/Parser/user-state.h +++ b/flang/include/flang/Parser/user-state.h @@ -26,7 +26,7 @@ namespace Fortran::parser { -class CookedSource; +class AllCookedSources; class ParsingLog; class ParseState; @@ -34,10 +34,11 @@ class Success {}; // for when one must return something that's present class UserState { public: - UserState(const CookedSource &cooked, common::LanguageFeatureControl features) - : cooked_{cooked}, features_{features} {} + UserState(const AllCookedSources &allCooked, + common::LanguageFeatureControl features) + : allCooked_{allCooked}, features_{features} {} - const CookedSource &cooked() const { return cooked_; } + const AllCookedSources &allCooked() const { return allCooked_; } const common::LanguageFeatureControl &features() const { return features_; } llvm::raw_ostream *debugOutput() const { return debugOutput_; } @@ -89,7 +90,7 @@ public: } private: - const CookedSource &cooked_; + const AllCookedSources &allCooked_; llvm::raw_ostream *debugOutput_{nullptr}; diff --git a/flang/include/flang/Semantics/scope.h b/flang/include/flang/Semantics/scope.h index 5ebe5f3..853d7044 100644 --- a/flang/include/flang/Semantics/scope.h +++ b/flang/include/flang/Semantics/scope.h @@ -187,10 +187,6 @@ public: const DeclTypeSpec &MakeTypeStarType(); const DeclTypeSpec &MakeClassStarType(); - // For modules read from module files, this is the stream of characters - // that are referenced by SourceName objects. - void set_chars(parser::CookedSource &); - std::size_t size() const { return size_; } void set_size(std::size_t size) { size_ = size; } std::size_t alignment() const { return alignment_; } @@ -245,7 +241,6 @@ private: mapType crayPointers_; std::map> submodules_; std::list declTypeSpecs_; - std::string chars_; std::optional importKind_; std::set importNames_; DerivedTypeSpec *derivedTypeSpec_{nullptr}; // dTS->scope() == this diff --git a/flang/include/flang/Semantics/semantics.h b/flang/include/flang/Semantics/semantics.h index 3c7ba98..4c2c0e7 100644 --- a/flang/include/flang/Semantics/semantics.h +++ b/flang/include/flang/Semantics/semantics.h @@ -30,7 +30,7 @@ class IntrinsicTypeDefaultKinds; namespace Fortran::parser { struct Name; struct Program; -class CookedSource; +class AllCookedSources; struct AssociateConstruct; struct BlockConstruct; struct CaseConstruct; @@ -60,7 +60,7 @@ using ConstructStack = std::vector; class SemanticsContext { public: SemanticsContext(const common::IntrinsicTypeDefaultKinds &, - const common::LanguageFeatureControl &, parser::AllSources &); + const common::LanguageFeatureControl &, parser::AllCookedSources &); ~SemanticsContext(); const common::IntrinsicTypeDefaultKinds &defaultKinds() const { @@ -89,7 +89,7 @@ public: Scope &globalScope() { return globalScope_; } parser::Messages &messages() { return messages_; } evaluate::FoldingContext &foldingContext() { return foldingContext_; } - parser::AllSources &allSources() { return allSources_; } + parser::AllCookedSources &allCookedSources() { return allCookedSources_; } SemanticsContext &set_location( const std::optional &location) { @@ -179,7 +179,7 @@ private: const common::IntrinsicTypeDefaultKinds &defaultKinds_; const common::LanguageFeatureControl languageFeatures_; - parser::AllSources &allSources_; + parser::AllCookedSources &allCookedSources_; std::optional location_; std::vector searchDirectories_; std::string moduleDirectory_{"."s}; @@ -204,8 +204,8 @@ private: class Semantics { public: explicit Semantics(SemanticsContext &context, parser::Program &program, - parser::CookedSource &cooked, bool debugModuleWriter = false) - : context_{context}, program_{program}, cooked_{cooked} { + const parser::CookedSource &cooked, bool debugModuleWriter = false) + : context_{context}, program_{program} { context.set_debugModuleWriter(debugModuleWriter); context.globalScope().AddSourceRange(parser::CharBlock{cooked.data()}); } @@ -223,7 +223,6 @@ public: private: SemanticsContext &context_; parser::Program &program_; - const parser::CookedSource &cooked_; }; // Base class for semantics checkers. diff --git a/flang/lib/Parser/debug-parser.cpp b/flang/lib/Parser/debug-parser.cpp index dbcc64f1..af5da09 100644 --- a/flang/lib/Parser/debug-parser.cpp +++ b/flang/lib/Parser/debug-parser.cpp @@ -18,9 +18,9 @@ std::optional DebugParser::Parse(ParseState &state) const { std::string note{str_, length_}; Message message{state.GetLocation(), "parser debug: %s"_en_US, note}; message.SetContext(state.context().get()); - message.Emit(*out, ustate->cooked(), true); + message.Emit(*out, ustate->allCooked(), true); } } - return {Success{}}; + return Success{}; } } // namespace Fortran::parser diff --git a/flang/lib/Parser/instrumented-parser.cpp b/flang/lib/Parser/instrumented-parser.cpp index 765d292..6687aa1 100644 --- a/flang/lib/Parser/instrumented-parser.cpp +++ b/flang/lib/Parser/instrumented-parser.cpp @@ -63,14 +63,15 @@ void ParsingLog::Note(const char *at, const MessageFixedText &tag, bool pass, } } -void ParsingLog::Dump(llvm::raw_ostream &o, const CookedSource &cooked) const { +void ParsingLog::Dump( + llvm::raw_ostream &o, const AllCookedSources &allCooked) const { for (const auto &posLog : perPos_) { const char *at{reinterpret_cast(posLog.first)}; for (const auto &tagLog : posLog.second.perTag) { - Message{at, tagLog.first}.Emit(o, cooked, true); + Message{at, tagLog.first}.Emit(o, allCooked, true); auto &entry{tagLog.second}; o << " " << (entry.pass ? "pass" : "fail") << " " << entry.count << '\n'; - entry.messages.Emit(o, cooked); + entry.messages.Emit(o, allCooked); } } } diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp index 87594d6..6819ee4d 100644 --- a/flang/lib/Parser/message.cpp +++ b/flang/lib/Parser/message.cpp @@ -165,43 +165,43 @@ std::string Message::ToString() const { text_); } -void Message::ResolveProvenances(const CookedSource &cooked) { +void Message::ResolveProvenances(const AllCookedSources &allCooked) { if (CharBlock * cb{std::get_if(&location_)}) { if (std::optional resolved{ - cooked.GetProvenanceRange(*cb)}) { + allCooked.GetProvenanceRange(*cb)}) { location_ = *resolved; } } if (Message * attachment{attachment_.get()}) { - attachment->ResolveProvenances(cooked); + attachment->ResolveProvenances(allCooked); } } std::optional Message::GetProvenanceRange( - const CookedSource &cooked) const { + const AllCookedSources &allCooked) const { return std::visit( common::visitors{ - [&](CharBlock cb) { return cooked.GetProvenanceRange(cb); }, + [&](CharBlock cb) { return allCooked.GetProvenanceRange(cb); }, [](const ProvenanceRange &pr) { return std::make_optional(pr); }, }, location_); } -void Message::Emit(llvm::raw_ostream &o, const CookedSource &cooked, +void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked, bool echoSourceLine) const { - std::optional provenanceRange{GetProvenanceRange(cooked)}; + std::optional provenanceRange{GetProvenanceRange(allCooked)}; std::string text; if (IsFatal()) { text += "error: "; } text += ToString(); - const AllSources &sources{cooked.allSources()}; + const AllSources &sources{allCooked.allSources()}; sources.EmitMessage(o, provenanceRange, text, echoSourceLine); if (attachmentIsContext_) { for (const Message *context{attachment_.get()}; context; context = context->attachment_.get()) { std::optional contextProvenance{ - context->GetProvenanceRange(cooked)}; + context->GetProvenanceRange(allCooked)}; text = "in the context: "; text += context->ToString(); // TODO: don't echo the source lines of a context when it's the @@ -213,7 +213,7 @@ void Message::Emit(llvm::raw_ostream &o, const CookedSource &cooked, } else { for (const Message *attachment{attachment_.get()}; attachment; attachment = attachment->attachment_.get()) { - sources.EmitMessage(o, attachment->GetProvenanceRange(cooked), + sources.EmitMessage(o, attachment->GetProvenanceRange(allCooked), attachment->ToString(), echoSourceLine); } } @@ -300,13 +300,13 @@ void Messages::Copy(const Messages &that) { } } -void Messages::ResolveProvenances(const CookedSource &cooked) { +void Messages::ResolveProvenances(const AllCookedSources &allCooked) { for (Message &m : messages_) { - m.ResolveProvenances(cooked); + m.ResolveProvenances(allCooked); } } -void Messages::Emit(llvm::raw_ostream &o, const CookedSource &cooked, +void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked, bool echoSourceLines) const { std::vector sorted; for (const auto &msg : messages_) { @@ -315,7 +315,7 @@ void Messages::Emit(llvm::raw_ostream &o, const CookedSource &cooked, std::stable_sort(sorted.begin(), sorted.end(), [](const Message *x, const Message *y) { return x->SortBefore(*y); }); for (const Message *msg : sorted) { - msg->Emit(o, cooked, echoSourceLines); + msg->Emit(o, allCooked, echoSourceLines); } } diff --git a/flang/lib/Parser/parsing.cpp b/flang/lib/Parser/parsing.cpp index b77242a..819f3cf 100644 --- a/flang/lib/Parser/parsing.cpp +++ b/flang/lib/Parser/parsing.cpp @@ -17,12 +17,12 @@ namespace Fortran::parser { -Parsing::Parsing(AllSources &s) : cooked_{s} {} +Parsing::Parsing(AllCookedSources &allCooked) : allCooked_{allCooked} {} Parsing::~Parsing() {} const SourceFile *Parsing::Prescan(const std::string &path, Options options) { options_ = options; - AllSources &allSources{cooked_.allSources()}; + AllSources &allSources{allCooked_.allSources()}; if (options.isModuleFile) { for (const auto &path : options.searchDirectories) { allSources.PushSearchPathDirectory(path); @@ -63,7 +63,9 @@ const SourceFile *Parsing::Prescan(const std::string &path, Options options) { preprocessor.Undefine(predef.first); } } - Prescanner prescanner{messages_, cooked_, preprocessor, options.features}; + currentCooked_ = &allCooked_.NewCookedSource(); + Prescanner prescanner{ + messages_, *currentCooked_, allSources, preprocessor, options.features}; prescanner.set_fixedForm(options.isFixedForm) .set_fixedFormColumnLimit(options.fixedFormColumns) .AddCompilerDirectiveSentinel("dir$"); @@ -77,21 +79,21 @@ const SourceFile *Parsing::Prescan(const std::string &path, Options options) { ProvenanceRange range{allSources.AddIncludedFile( *sourceFile, ProvenanceRange{}, options.isModuleFile)}; prescanner.Prescan(range); - if (cooked_.BufferedBytes() == 0 && !options.isModuleFile) { + if (currentCooked_->BufferedBytes() == 0 && !options.isModuleFile) { // Input is empty. Append a newline so that any warning // message about nonstandard usage will have provenance. - cooked_.Put('\n', range.start()); + currentCooked_->Put('\n', range.start()); } - cooked_.Marshal(); + currentCooked_->Marshal(allSources); if (options.needProvenanceRangeToCharBlockMappings) { - cooked_.CompileProvenanceRangeToOffsetMappings(); + currentCooked_->CompileProvenanceRangeToOffsetMappings(allSources); } return sourceFile; } void Parsing::DumpCookedChars(llvm::raw_ostream &out) const { - UserState userState{cooked_, common::LanguageFeatureControl{}}; - ParseState parseState{cooked_}; + UserState userState{allCooked_, common::LanguageFeatureControl{}}; + ParseState parseState{cooked()}; parseState.set_inFixedForm(options_.isFixedForm).set_userState(&userState); while (std::optional p{parseState.GetNextChar()}) { out << **p; @@ -99,19 +101,19 @@ void Parsing::DumpCookedChars(llvm::raw_ostream &out) const { } void Parsing::DumpProvenance(llvm::raw_ostream &out) const { - cooked_.Dump(out); + allCooked_.Dump(out); } void Parsing::DumpParsingLog(llvm::raw_ostream &out) const { - log_.Dump(out, cooked_); + log_.Dump(out, allCooked_); } void Parsing::Parse(llvm::raw_ostream &out) { - UserState userState{cooked_, options_.features}; + UserState userState{allCooked_, options_.features}; userState.set_debugOutput(out) .set_instrumentedParse(options_.instrumentedParse) .set_log(&log_); - ParseState parseState{cooked_}; + ParseState parseState{cooked()}; parseState.set_inFixedForm(options_.isFixedForm).set_userState(&userState); parseTree_ = program.Parse(parseState); CHECK( diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp index 5e6f137..8e8e57c 100644 --- a/flang/lib/Parser/prescan.cpp +++ b/flang/lib/Parser/prescan.cpp @@ -26,14 +26,16 @@ using common::LanguageFeature; static constexpr int maxPrescannerNesting{100}; Prescanner::Prescanner(Messages &messages, CookedSource &cooked, - Preprocessor &preprocessor, common::LanguageFeatureControl lfc) - : messages_{messages}, cooked_{cooked}, preprocessor_{preprocessor}, - features_{lfc}, encoding_{cooked.allSources().encoding()} {} + AllSources &allSources, Preprocessor &preprocessor, + common::LanguageFeatureControl lfc) + : messages_{messages}, cooked_{cooked}, allSources_{allSources}, + preprocessor_{preprocessor}, features_{lfc}, + encoding_{allSources_.encoding()} {} Prescanner::Prescanner(const Prescanner &that) : messages_{that.messages_}, cooked_{that.cooked_}, - preprocessor_{that.preprocessor_}, features_{that.features_}, - inFixedForm_{that.inFixedForm_}, + allSources_{that.allSources_}, preprocessor_{that.preprocessor_}, + features_{that.features_}, inFixedForm_{that.inFixedForm_}, fixedFormColumnLimit_{that.fixedFormColumnLimit_}, encoding_{that.encoding_}, prescannerNesting_{that.prescannerNesting_ + 1}, @@ -59,10 +61,10 @@ static void NormalizeCompilerDirectiveCommentMarker(TokenSequence &dir) { } void Prescanner::Prescan(ProvenanceRange range) { - AllSources &allSources{cooked_.allSources()}; startProvenance_ = range.start(); std::size_t offset{0}; - const SourceFile *source{allSources.GetSourceFile(startProvenance_, &offset)}; + const SourceFile *source{ + allSources_.GetSourceFile(startProvenance_, &offset)}; CHECK(source); start_ = source->content().data() + offset; limit_ = start_ + range.size(); @@ -84,7 +86,7 @@ void Prescanner::Prescan(ProvenanceRange range) { dir += "free"; } dir += '\n'; - TokenSequence tokens{dir, allSources.AddCompilerInsertion(dir).start()}; + TokenSequence tokens{dir, allSources_.AddCompilerInsertion(dir).start()}; tokens.Emit(cooked_); } } @@ -761,14 +763,13 @@ void Prescanner::FortranInclude(const char *firstQuote) { std::string buf; llvm::raw_string_ostream error{buf}; Provenance provenance{GetProvenance(nextLine_)}; - AllSources &allSources{cooked_.allSources()}; - const SourceFile *currentFile{allSources.GetSourceFile(provenance)}; + const SourceFile *currentFile{allSources_.GetSourceFile(provenance)}; if (currentFile) { - allSources.PushSearchPathDirectory(DirectoryName(currentFile->path())); + allSources_.PushSearchPathDirectory(DirectoryName(currentFile->path())); } - const SourceFile *included{allSources.Open(path, error)}; + const SourceFile *included{allSources_.Open(path, error)}; if (currentFile) { - allSources.PopSearchPathDirectory(); + allSources_.PopSearchPathDirectory(); } if (!included) { Say(provenance, "INCLUDE: %s"_err_en_US, error.str()); @@ -776,7 +777,7 @@ void Prescanner::FortranInclude(const char *firstQuote) { ProvenanceRange includeLineRange{ provenance, static_cast(p - nextLine_)}; ProvenanceRange fileRange{ - allSources.AddIncludedFile(*included, includeLineRange)}; + allSources_.AddIncludedFile(*included, includeLineRange)}; Prescanner{*this}.set_encoding(included->encoding()).Prescan(fileRange); } } diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h index 0b5b647..ab56ed4 100644 --- a/flang/lib/Parser/prescan.h +++ b/flang/lib/Parser/prescan.h @@ -33,7 +33,7 @@ class Preprocessor; class Prescanner { public: - Prescanner(Messages &, CookedSource &, Preprocessor &, + Prescanner(Messages &, CookedSource &, AllSources &, Preprocessor &, common::LanguageFeatureControl); Prescanner(const Prescanner &); @@ -65,10 +65,7 @@ public: Provenance GetCurrentProvenance() const { return GetProvenance(at_); } template Message &Say(A &&...a) { - Message &m{messages_.Say(std::forward(a)...)}; - std::optional range{m.GetProvenanceRange(cooked_)}; - CHECK(!range || cooked_.IsValid(*range)); - return m; + return messages_.Say(std::forward(a)...); } private: @@ -124,7 +121,7 @@ private: } void EmitInsertedChar(TokenSequence &tokens, char ch) { - Provenance provenance{cooked_.allSources().CompilerInsertionProvenance(ch)}; + Provenance provenance{allSources_.CompilerInsertionProvenance(ch)}; tokens.PutNextTokenChar(ch, provenance); } @@ -184,6 +181,7 @@ private: Messages &messages_; CookedSource &cooked_; + AllSources &allSources_; Preprocessor &preprocessor_; common::LanguageFeatureControl features_; bool inFixedForm_{false}; @@ -222,9 +220,9 @@ private: bool skipLeadingAmpersand_{false}; const Provenance spaceProvenance_{ - cooked_.allSources().CompilerInsertionProvenance(' ')}; + allSources_.CompilerInsertionProvenance(' ')}; const Provenance backslashProvenance_{ - cooked_.allSources().CompilerInsertionProvenance('\\')}; + allSources_.CompilerInsertionProvenance('\\')}; // To avoid probing the set of active compiler directive sentinel strings // on every comment line, they're checked first with a cheap Bloom filter. diff --git a/flang/lib/Parser/provenance.cpp b/flang/lib/Parser/provenance.cpp index 73e0f71..bcb871b 100644 --- a/flang/lib/Parser/provenance.cpp +++ b/flang/lib/Parser/provenance.cpp @@ -400,12 +400,9 @@ const AllSources::Origin &AllSources::MapToOrigin(Provenance at) const { return origin_[low]; } -CookedSource::CookedSource(AllSources &s) : allSources_{s} {} -CookedSource::~CookedSource() {} - std::optional CookedSource::GetProvenanceRange( CharBlock cookedRange) const { - if (!IsValid(cookedRange)) { + if (!Contains(cookedRange)) { return std::nullopt; } ProvenanceRange first{provenanceMap_.Map(cookedRange.begin() - &data_[0])}; @@ -416,34 +413,6 @@ std::optional CookedSource::GetProvenanceRange( return {ProvenanceRange{first.start(), last.start() - first.start()}}; } -std::optional CookedSource::GetCharBlockFromLineAndColumns( - int line, int startColumn, int endColumn) const { - // 2nd column is exclusive, meaning it is target column + 1. - CHECK(line > 0 && startColumn > 0 && endColumn > 0); - CHECK(startColumn < endColumn); - auto provenanceStart{allSources_.GetFirstFileProvenance().value().start()}; - if (auto sourceFile{allSources_.GetSourceFile(provenanceStart)}) { - CHECK(line <= static_cast(sourceFile->lines())); - return GetCharBlock(ProvenanceRange(sourceFile->GetLineStartOffset(line) + - provenanceStart.offset() + startColumn - 1, - endColumn - startColumn)); - } - return std::nullopt; -} - -std::optional> -CookedSource::GetSourcePositionRange(CharBlock cookedRange) const { - if (auto range{GetProvenanceRange(cookedRange)}) { - if (auto firstOffset{allSources_.GetSourcePosition(range->start())}) { - if (auto secondOffset{ - allSources_.GetSourcePosition(range->start() + range->size())}) { - return std::pair{*firstOffset, *secondOffset}; - } - } - } - return std::nullopt; -} - std::optional CookedSource::GetCharBlock( ProvenanceRange range) const { CHECK(!invertedMap_.empty() && @@ -457,16 +426,17 @@ std::optional CookedSource::GetCharBlock( std::size_t CookedSource::BufferedBytes() const { return buffer_.bytes(); } -void CookedSource::Marshal() { +void CookedSource::Marshal(AllSources &allSources) { CHECK(provenanceMap_.SizeInBytes() == buffer_.bytes()); - provenanceMap_.Put(allSources_.AddCompilerInsertion("(after end of source)")); + provenanceMap_.Put(allSources.AddCompilerInsertion("(after end of source)")); data_ = buffer_.Marshal(); buffer_.clear(); } -void CookedSource::CompileProvenanceRangeToOffsetMappings() { +void CookedSource::CompileProvenanceRangeToOffsetMappings( + AllSources &allSources) { if (invertedMap_.empty()) { - invertedMap_ = provenanceMap_.Invert(allSources_); + invertedMap_ = provenanceMap_.Invert(allSources); } } @@ -534,12 +504,73 @@ llvm::raw_ostream &AllSources::Dump(llvm::raw_ostream &o) const { } llvm::raw_ostream &CookedSource::Dump(llvm::raw_ostream &o) const { - o << "CookedSource:\n"; - allSources_.Dump(o); o << "CookedSource::provenanceMap_:\n"; provenanceMap_.Dump(o); o << "CookedSource::invertedMap_:\n"; invertedMap_.Dump(o); return o; } + +AllCookedSources::AllCookedSources(AllSources &s) : allSources_{s} {} +AllCookedSources::~AllCookedSources() {} + +CookedSource &AllCookedSources::NewCookedSource() { + return cooked_.emplace_back(); +} + +std::optional AllCookedSources::GetProvenanceRange( + CharBlock cb) const { + if (const CookedSource * c{Find(cb)}) { + return c->GetProvenanceRange(cb); + } else { + return std::nullopt; + } +} + +std::optional AllCookedSources::GetCharBlockFromLineAndColumns( + int line, int startColumn, int endColumn) const { + // 2nd column is exclusive, meaning it is target column + 1. + CHECK(line > 0 && startColumn > 0 && endColumn > 0); + CHECK(startColumn < endColumn); + auto provenanceStart{allSources_.GetFirstFileProvenance().value().start()}; + if (auto sourceFile{allSources_.GetSourceFile(provenanceStart)}) { + CHECK(line <= static_cast(sourceFile->lines())); + return GetCharBlock(ProvenanceRange(sourceFile->GetLineStartOffset(line) + + provenanceStart.offset() + startColumn - 1, + endColumn - startColumn)); + } + return std::nullopt; +} + +std::optional> +AllCookedSources::GetSourcePositionRange(CharBlock cookedRange) const { + if (auto range{GetProvenanceRange(cookedRange)}) { + if (auto firstOffset{allSources_.GetSourcePosition(range->start())}) { + if (auto secondOffset{ + allSources_.GetSourcePosition(range->start() + range->size())}) { + return std::pair{*firstOffset, *secondOffset}; + } + } + } + return std::nullopt; +} + +std::optional AllCookedSources::GetCharBlock( + ProvenanceRange range) const { + for (const auto &c : cooked_) { + if (auto result{c.GetCharBlock(range)}) { + return result; + } + } + return nullptr; +} + +void AllCookedSources::Dump(llvm::raw_ostream &o) const { + o << "AllSources:\n"; + allSources_.Dump(o); + for (const auto &c : cooked_) { + c.Dump(o); + } +} + } // namespace Fortran::parser diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp index 6fa59f0..ef62a94 100644 --- a/flang/lib/Semantics/mod-file.cpp +++ b/flang/lib/Semantics/mod-file.cpp @@ -751,7 +751,7 @@ Scope *ModFileReader::Read(const SourceName &name, Scope *ancestor) { return it->second->scope(); } } - parser::Parsing parsing{context_.allSources()}; + parser::Parsing parsing{context_.allCookedSources()}; parser::Options options; options.isModuleFile = true; options.features.Enable(common::LanguageFeature::BackslashEscapes); @@ -796,7 +796,6 @@ Scope *ModFileReader::Read(const SourceName &name, Scope *ancestor) { } auto &modSymbol{*it->second}; modSymbol.set(Symbol::Flag::ModFile); - modSymbol.scope()->set_chars(parsing.cooked()); return modSymbol.scope(); } diff --git a/flang/lib/Semantics/scope.cpp b/flang/lib/Semantics/scope.cpp index a2a9e1d..c7635c0 100644 --- a/flang/lib/Semantics/scope.cpp +++ b/flang/lib/Semantics/scope.cpp @@ -217,14 +217,6 @@ DeclTypeSpec &Scope::MakeDerivedType( return declTypeSpecs_.emplace_back(category, std::move(spec)); } -void Scope::set_chars(parser::CookedSource &cooked) { - CHECK(kind_ == Kind::Module); - CHECK(parent_.IsGlobal() || parent_.IsModuleFile()); - CHECK(DEREF(symbol_).test(Symbol::Flag::ModFile)); - // TODO: Preserve the CookedSource rather than acquiring its string. - chars_ = cooked.AcquireData(); -} - Scope::ImportKind Scope::GetImportKind() const { if (importKind_) { return *importKind_; diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp index af5b120..b5b7802 100644 --- a/flang/lib/Semantics/semantics.cpp +++ b/flang/lib/Semantics/semantics.cpp @@ -181,9 +181,9 @@ static bool PerformStatementSemantics( SemanticsContext::SemanticsContext( const common::IntrinsicTypeDefaultKinds &defaultKinds, const common::LanguageFeatureControl &languageFeatures, - parser::AllSources &allSources) + parser::AllCookedSources &allCookedSources) : defaultKinds_{defaultKinds}, languageFeatures_{languageFeatures}, - allSources_{allSources}, + allCookedSources_{allCookedSources}, intrinsics_{evaluate::IntrinsicProcTable::Configure(defaultKinds_)}, foldingContext_{ parser::ContextualMessages{&messages_}, defaultKinds_, intrinsics_} {} @@ -351,7 +351,7 @@ bool Semantics::Perform() { } void Semantics::EmitMessages(llvm::raw_ostream &os) const { - context_.messages().Emit(os, cooked_); + context_.messages().Emit(os, context_.allCookedSources()); } void Semantics::DumpSymbols(llvm::raw_ostream &os) { @@ -361,9 +361,10 @@ void Semantics::DumpSymbols(llvm::raw_ostream &os) { void Semantics::DumpSymbolsSources(llvm::raw_ostream &os) const { NameToSymbolMap symbols; GetSymbolNames(context_.globalScope(), symbols); + const parser::AllCookedSources &allCooked{context_.allCookedSources()}; for (const auto &pair : symbols) { const Symbol &symbol{pair.second}; - if (auto sourceInfo{cooked_.GetSourcePositionRange(symbol.name())}) { + if (auto sourceInfo{allCooked.GetSourcePositionRange(symbol.name())}) { os << symbol.name().ToString() << ": " << sourceInfo->first.file.path() << ", " << sourceInfo->first.line << ", " << sourceInfo->first.column << "-" << sourceInfo->second.column << "\n"; diff --git a/flang/test/Semantics/getsymbols02.f90 b/flang/test/Semantics/getsymbols02.f90 index 1eed3e9..80b7651 100644 --- a/flang/test/Semantics/getsymbols02.f90 +++ b/flang/test/Semantics/getsymbols02.f90 @@ -10,5 +10,5 @@ ENDPROGRAM ! RUN: %f18 -fparse-only %S/Inputs/getsymbols02-a.f90 ! RUN: %f18 -fparse-only %S/Inputs/getsymbols02-b.f90 ! RUN: %f18 -fget-symbols-sources -fparse-only %s 2>&1 | FileCheck %s -! CHECK: callget5: mm2b -! CHECK: get5: mm2a +! CHECK: callget5: ./mm2b.mod, +! CHECK: get5: ./mm2a.mod, diff --git a/flang/tools/f18-parse-demo/f18-parse-demo.cpp b/flang/tools/f18-parse-demo/f18-parse-demo.cpp index 60303aa..4ccc65e 100644 --- a/flang/tools/f18-parse-demo/f18-parse-demo.cpp +++ b/flang/tools/f18-parse-demo/f18-parse-demo.cpp @@ -160,14 +160,15 @@ std::string CompileFortran( } options.searchDirectories = driver.searchDirectories; Fortran::parser::AllSources allSources; - Fortran::parser::Parsing parsing{allSources}; + Fortran::parser::AllCookedSources allCookedSources{allSources}; + Fortran::parser::Parsing parsing{allCookedSources}; auto start{CPUseconds()}; parsing.Prescan(path, options); if (!parsing.messages().empty() && (driver.warningsAreErrors || parsing.messages().AnyFatalError())) { llvm::errs() << driver.prefix << "could not scan " << path << '\n'; - parsing.messages().Emit(llvm::errs(), parsing.cooked()); + parsing.messages().Emit(llvm::errs(), parsing.allCooked()); exitStatus = EXIT_FAILURE; return {}; } @@ -191,7 +192,7 @@ std::string CompileFortran( } parsing.ClearLog(); - parsing.messages().Emit(llvm::errs(), parsing.cooked()); + parsing.messages().Emit(llvm::errs(), parsing.allCooked()); if (!parsing.consumedWholeFile()) { parsing.EmitMessage(llvm::errs(), parsing.finalRestingPlace(), "parser FAIL (final position)"); diff --git a/flang/tools/f18/f18.cpp b/flang/tools/f18/f18.cpp index 156c233..a33a167 100644 --- a/flang/tools/f18/f18.cpp +++ b/flang/tools/f18/f18.cpp @@ -188,9 +188,10 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options, DriverOptions &driver, const Fortran::common::IntrinsicTypeDefaultKinds &defaultKinds) { Fortran::parser::AllSources allSources; + Fortran::parser::AllCookedSources allCookedSources{allSources}; allSources.set_encoding(driver.encoding); Fortran::semantics::SemanticsContext semanticsContext{ - defaultKinds, options.features, allSources}; + defaultKinds, options.features, allCookedSources}; semanticsContext.set_moduleDirectory(driver.moduleDirectory) .set_moduleFileSuffix(driver.moduleFileSuffix) .set_searchDirectories(driver.searchDirectories) @@ -204,12 +205,12 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options, } } options.searchDirectories = driver.searchDirectories; - Fortran::parser::Parsing parsing{semanticsContext.allSources()}; + Fortran::parser::Parsing parsing{allCookedSources}; parsing.Prescan(path, options); if (!parsing.messages().empty() && (driver.warningsAreErrors || parsing.messages().AnyFatalError())) { llvm::errs() << driver.prefix << "could not scan " << path << '\n'; - parsing.messages().Emit(llvm::errs(), parsing.cooked()); + parsing.messages().Emit(llvm::errs(), allCookedSources); exitStatus = EXIT_FAILURE; return {}; } @@ -218,7 +219,7 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options, return {}; } if (driver.dumpCookedChars) { - parsing.messages().Emit(llvm::errs(), parsing.cooked()); + parsing.messages().Emit(llvm::errs(), allCookedSources); parsing.DumpCookedChars(llvm::outs()); return {}; } @@ -228,7 +229,7 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options, return {}; } parsing.ClearLog(); - parsing.messages().Emit(llvm::errs(), parsing.cooked()); + parsing.messages().Emit(llvm::errs(), allCookedSources); if (!parsing.consumedWholeFile()) { parsing.EmitMessage(llvm::errs(), parsing.finalRestingPlace(), "parser FAIL (final position)"); @@ -274,7 +275,7 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options, return {}; } if (driver.getDefinition) { - if (auto cb{parsing.cooked().GetCharBlockFromLineAndColumns( + if (auto cb{allCookedSources.GetCharBlockFromLineAndColumns( driver.getDefinitionArgs.line, driver.getDefinitionArgs.startColumn, driver.getDefinitionArgs.endColumn)}) { @@ -283,7 +284,7 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options, llvm::errs() << "Found symbol name: " << symbol->name().ToString() << "\n"; if (auto sourceInfo{ - parsing.cooked().GetSourcePositionRange(symbol->name())}) { + allCookedSources.GetSourcePositionRange(symbol->name())}) { llvm::outs() << symbol->name().ToString() << ": " << sourceInfo->first.file.path() << ", " << sourceInfo->first.line << ", " diff --git a/flang/unittests/Evaluate/intrinsics.cpp b/flang/unittests/Evaluate/intrinsics.cpp index 3b98059..4f2a21d 100644 --- a/flang/unittests/Evaluate/intrinsics.cpp +++ b/flang/unittests/Evaluate/intrinsics.cpp @@ -22,9 +22,9 @@ public: } void Save(const std::string &s) { offsets_[s] = cooked_.Put(s); - cooked_.PutProvenance(cooked_.allSources().AddCompilerInsertion(s)); + cooked_.PutProvenance(allSources_.AddCompilerInsertion(s)); } - void Marshal() { cooked_.Marshal(); } + void Marshal() { cooked_.Marshal(allSources_); } parser::CharBlock operator()(const std::string &s) { return {cooked_.data().data() + offsets_[s], s.size()}; } @@ -32,12 +32,13 @@ public: return parser::ContextualMessages{cooked_.data(), &buffer}; } void Emit(llvm::raw_ostream &o, const parser::Messages &messages) { - messages.Emit(o, cooked_); + messages.Emit(o, allCookedSources_); } private: parser::AllSources allSources_; - parser::CookedSource cooked_{allSources_}; + parser::AllCookedSources allCookedSources_{allSources_}; + parser::CookedSource &cooked_{allCookedSources_.NewCookedSource()}; std::map offsets_; }; -- 2.7.4