[flang] Fix fixed form scanning, get tests passing again.

Original-commit: flang-compiler/f18@2c8ff18e988143f81b0e3216386caabc1a6feeb4
Reviewed-on: https://github.com/flang-compiler/f18/pull/9
Tree-same-pre-rewrite: false

diff --git a/flang/lib/parser/preprocessor.cc b/flang/lib/parser/preprocessor.cc
index b4fdbf0..5b21501 100644 (file)
--- a/flang/lib/parser/preprocessor.cc
+++ b/flang/lib/parser/preprocessor.cc
@@ -108,7 +108,7 @@ void TokenSequence::EmitWithCaseConversion(CookedSource *cooked) const {
       j = nextStart;
     }
   }
-  cooked->Put(provenances_);
+  cooked->PutProvenanceMappings(provenances_);
 }
 
 std::string TokenSequence::ToString() const {

diff --git a/flang/lib/parser/prescan.cc b/flang/lib/parser/prescan.cc
index 139ddf2..eddf483 100644 (file)
--- a/flang/lib/parser/prescan.cc
+++ b/flang/lib/parser/prescan.cc
@@ -10,18 +10,17 @@ namespace Fortran {
 namespace parser {
 
 Prescanner::Prescanner(Messages *messages, AllSources *allSources)
-  : messages_{messages}, allSources_{allSources}, preprocessor_{*this} {
-  std::string compilerInserts{" ,\"01"};
+  : messages_{messages}, allSources_{allSources}, start_{&(*allSources)[0]},
+    limit_{start_ + allSources->size()}, preprocessor_{*this} {
+  std::string compilerInserts{" ,\"01\n"};
   ProvenanceRange range{allSources->AddCompilerInsertion(compilerInserts)};
   for (size_t j{0}; j < compilerInserts.size(); ++j) {
     compilerInsertionProvenance_[compilerInserts[j]] = range.start + j;
   }
+  newlineProvenance_ = CompilerInsertionProvenance('\n');
 }
 
 CookedSource Prescanner::Prescan() {
-  startProvenance_ = 0;
-  start_ = &(*allSources_)[0];
-  limit_ = start_ + allSources_->size();
   lineStart_ = start_;
   BeginSourceLine(start_);
   TokenSequence tokens, preprocessed;
@@ -39,7 +38,7 @@ CookedSource Prescanner::Prescan() {
     while (NextToken(&tokens)) {
     }
     if (preprocessor_.MacroReplacement(tokens, &preprocessed)) {
-      EmitChar(&preprocessed, '\n');
+      preprocessed.PutNextTokenChar('\n', newlineProvenance_);
       preprocessed.CloseToken();
       if (IsFixedFormCommentLine(preprocessed.data()) ||
           IsFreeFormComment(preprocessed.data())) {
@@ -53,7 +52,7 @@ CookedSource Prescanner::Prescan() {
       tokens.EmitWithCaseConversion(&cooked);
     }
     tokens.clear();
-    cooked.Put('\n', 0);
+    cooked.Put('\n', newlineProvenance_);
     PayNewlineDebt(&cooked);
   }
   PayNewlineDebt(&cooked);
@@ -120,7 +119,7 @@ void Prescanner::LabelField(TokenSequence *token) {
 }
 
 void Prescanner::NextChar() {
-  // CHECK(*at_ != '\n');
+  CHECK(*at_ != '\n');  // TODO pmk
   ++at_;
   ++column_;
   if (inPreprocessorDirective_) {
@@ -169,6 +168,7 @@ static inline bool IsNameChar(char ch) {
 }
 
 bool Prescanner::NextToken(TokenSequence *tokens) {
+  CHECK(at_ > start_ && at_ < limit_);  // TODO pmk
   if (inFixedForm_) {
     SkipSpaces();
   } else if (*at_ == ' ' || *at_ == '\t') {
@@ -500,7 +500,7 @@ bool Prescanner::FreeFormContinuation() {
 
 void Prescanner::PayNewlineDebt(CookedSource *cooked) {
   for (; newlineDebt_ > 0; --newlineDebt_) {
-    cooked->Put('\n', 0);
+    cooked->Put('\n', newlineProvenance_);
   }
 }
 }  // namespace parser

diff --git a/flang/lib/parser/prescan.h b/flang/lib/parser/prescan.h
index a4c22b3..b5a28c3 100644 (file)
--- a/flang/lib/parser/prescan.h
+++ b/flang/lib/parser/prescan.h
@@ -65,7 +65,7 @@ private:
   }
 
   Provenance GetProvenance(const char *sourceChar) const {
-    return startProvenance_ + sourceChar - start_;
+    return startProvenance_ /*TODO pmk rm?*/ + sourceChar - start_;
   }
 
   void EmitChar(TokenSequence *tokens, char ch) {
@@ -99,7 +99,7 @@ private:
   Messages *messages_;
   AllSources *allSources_;
 
-  Provenance startProvenance_;
+  Provenance startProvenance_{0};
   const char *start_{nullptr};  // beginning of sourceFile_ content
   const char *limit_{nullptr};  // first address after end of source
   const char *at_{nullptr};  // next character to process; < lineStart_
@@ -119,6 +119,7 @@ private:
   int delimiterNesting_{0};
   Preprocessor preprocessor_;
   std::map<char, Provenance> compilerInsertionProvenance_;
+  Provenance newlineProvenance_{0};
 };
 }  // namespace parser
 }  // namespace Fortran

diff --git a/flang/lib/parser/provenance.h b/flang/lib/parser/provenance.h
index b70e23a..d172396 100644 (file)
--- a/flang/lib/parser/provenance.h
+++ b/flang/lib/parser/provenance.h
@@ -126,7 +126,9 @@ public:
     buffer_.Put(&ch, 1);
     provenanceMap_.Put(ProvenanceRange{p, 1});
   }
-  void Put(const OffsetToProvenanceMappings &pm) { provenanceMap_.Put(pm); }
+  void PutProvenanceMappings(const OffsetToProvenanceMappings &pm) {
+    provenanceMap_.Put(pm);
+  }
   void Marshal();  // marshalls all text into one contiguous block
 
 private: