From: Tim Keith <tkeith@nvidia.com>
Date: Wed, 25 Jul 2018 13:55:11 +0000 (-0700)
Subject: [flang] Implement reading of module files
X-Git-Tag: llvmorg-12-init~9537^2~2335
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f62f8b655d778ca1ed48e9161ea1405f54822ce5;p=platform%2Fupstream%2Fllvm.git

[flang] Implement reading of module files

When a use-stmt is encountered for a module that isn't in the global
scope, search for and read the appropriate `.mod` file. To perform the
search, pass the search directories in to ResolveNames.

For modules that were read from `.mod` files, we have to keep the cooked
source from being deleted so that the names so that references to names
stay valid. So we store the cooked source in the Scope of the module as
a `unique_ptr`.

Add `Symbol::Flag::ModFile` to distinguish module symbols that were read
from a `.mod` file rather than from the current compilation. Use it to
prevent writing those back out.

Fix test_errors.sh to run the compiler in the temp subdirectory --
otherwise tests could be affected by `.mod` files left from previous
tests.

Original-commit: flang-compiler/f18@207065999ce09ca7361aeb0262d1b8bf8cf4b99e
Reviewed-on: https://github.com/flang-compiler/f18/pull/145
---

diff --git a/flang/documentation/mod-files.md b/flang/documentation/mod-files.md
index 032894e..367cd4c 100644
--- a/flang/documentation/mod-files.md
+++ b/flang/documentation/mod-files.md
@@ -102,6 +102,11 @@ For PGI, `-I` specifies directories to search for include files and module
 files. `-module` specifics a directory to write module files in as well as to
 search for them. gfortran is similar except it uses `-J` instead of `-module`.
 
+The search order for module files is:
+1. The `-module` directory (Note: for gfortran the `-J` directory is not searched).
+2. The current directory
+3. The `-I` directories in the order they appear on the command line
+
 ### Writing module files
 
 When writing a module file, if the existing one matches what would be written,
diff --git a/flang/lib/semantics/mod-file.cc b/flang/lib/semantics/mod-file.cc
index e47259e..5f6b29c 100644
--- a/flang/lib/semantics/mod-file.cc
+++ b/flang/lib/semantics/mod-file.cc
@@ -15,7 +15,11 @@
 #include "mod-file.h"
 #include "scope.h"
 #include "symbol.h"
+#include "../parser/grammar.h"
 #include "../parser/message.h"
+#include "../parser/openmp-grammar.h"
+#include "../parser/preprocessor.h"
+#include "../parser/prescan.h"
 #include <algorithm>
 #include <cerrno>
 #include <fstream>
@@ -23,18 +27,28 @@
 #include <ostream>
 #include <set>
 #include <sstream>
+#include <sys/stat.h>
+#include <sys/types.h>
 #include <vector>
 
 namespace Fortran::semantics {
 
 using namespace parser::literals;
 
+// The extension used for module files.
+static constexpr auto extension{".mod"};
+// The initial characters of a file that identify it as a .mod file.
+static constexpr auto magic{"!mod$"};
+// Construct the path to a module file.
+static std::string ModFilePath(const std::string &, const std::string &);
+// Helpers for creating error messages.
+static parser::Message Error(
+    const SourceName &, parser::MessageFixedText, const std::string &);
+static parser::Message Error(const SourceName &, parser::MessageFixedText,
+    const std::string &, const std::string &);
+
 class ModFileWriter {
 public:
-  // The initial characters of a file that identify it as a .mod file.
-  static constexpr auto magic{"!mod$"};
-  static constexpr auto extension{".mod"};
-
   // The .mod file format version number.
   void set_version(int version) { version_ = version; }
   // The directory to write .mod files in.
@@ -103,7 +117,9 @@ bool ModFileWriter::WriteAll() {
   for (const auto &scope : Scope::globalScope.children()) {
     if (scope.kind() == Scope::Kind::Module) {
       auto &symbol{*scope.symbol()};  // symbol must be present for module
-      WriteOne(symbol);
+      if (!symbol.test(Symbol::Flag::ModFile)) {
+        WriteOne(symbol);
+      }
     }
   }
   return errors_.empty();
@@ -112,7 +128,7 @@ bool ModFileWriter::WriteAll() {
 bool ModFileWriter::WriteOne(const Symbol &modSymbol) {
   CHECK(modSymbol.has<ModuleDetails>());
   auto name{parser::ToLowerCaseLetters(modSymbol.name().ToString())};
-  std::string path{dir_ + '/' + name + extension};
+  std::string path{ModFilePath(dir_, name)};
   std::ofstream os{path};
   PutSymbols(*modSymbol.scope());
   std::string all{GetAsString(name)};
@@ -396,4 +412,103 @@ std::string ModFileWriter::CheckSum(const std::string &str) {
 
 void WriteModFiles() { ModFileWriter{}.WriteAll(std::cerr); }
 
+bool ModFileReader::Read(const SourceName &modName) {
+  auto path{FindModFile(modName)};
+  if (!path.has_value()) {
+    return false;
+  }
+  if (!Prescan(modName, *path)) {
+    return false;
+  }
+  parser::ParseState parseState{*cooked_};
+  auto parseTree{parser::program.Parse(parseState)};
+  if (!parseState.messages().empty()) {
+    errors_.emplace_back(modName,
+        parser::MessageFormattedText{
+            "Module file for '%s' is corrupt: %s"_err_en_US,
+            modName.ToString().data(), path->data()});
+    return false;
+  }
+  ResolveNames(*parseTree, *cooked_, directories_);
+  const auto &it{Scope::globalScope.find(modName)};
+  if (it == Scope::globalScope.end()) {
+    return false;
+  }
+  auto &modSymbol{*it->second};
+  modSymbol.scope()->set_cookedSource(std::move(cooked_));
+  modSymbol.set(Symbol::Flag::ModFile);
+  return true;
+}
+
+// Look for the .mod file for this module in the search directories.
+// Add to errors_ if not found.
+std::optional<std::string> ModFileReader::FindModFile(
+    const SourceName &modName) {
+  auto error{Error(modName, "Cannot find module file for '%s'"_err_en_US,
+      modName.ToString())};
+  for (auto &dir : directories_) {
+    std::string path{ModFilePath(dir, modName.ToString())};
+    std::ifstream ifstream{path};
+    if (!ifstream.good()) {
+      error.Attach(Error(
+          modName, "%s: %s"_en_US, path, std::string{std::strerror(errno)}));
+    } else {
+      std::string line;
+      ifstream >> line;
+      if (std::equal(line.begin(), line.end(), std::string{magic}.begin())) {
+        return path;  // success
+      }
+      error.Attach(Error(modName, "%s: Not a valid module file"_en_US, path));
+    }
+  }
+  errors_.push_back(error);
+  return std::nullopt;
+}
+
+bool ModFileReader::Prescan(
+    const SourceName &modName, const std::string &path) {
+  std::stringstream fileError;
+  const auto *sourceFile{allSources_.Open(path, &fileError)};
+  if (sourceFile == nullptr) {
+    errors_.push_back(
+        Error(modName, "Cannot read %s: %s"_err_en_US, path, fileError.str()));
+    return false;
+  }
+  parser::Preprocessor preprocessor{allSources_};
+  parser::Messages messages;
+  parser::Prescanner prescanner{messages, *cooked_, preprocessor, {}};
+  parser::ProvenanceRange range{
+      allSources_.AddIncludedFile(*sourceFile, parser::ProvenanceRange{})};
+  prescanner.Prescan(range);
+  if (!messages.empty()) {
+    errors_.push_back(
+        Error(modName, "Module file for '%s' is corrupt: %s"_err_en_US,
+            modName.ToString(), path));
+    return false;
+  }
+  cooked_->Marshal();
+  return true;
+}
+
+static std::string ModFilePath(
+    const std::string &dir, const std::string &modName) {
+  if (dir == "."s) {
+    return modName + extension;
+  } else {
+    return dir + '/' + modName + extension;
+  }
+}
+
+static parser::Message Error(const SourceName &location,
+    parser::MessageFixedText fixedText, const std::string &arg) {
+  return parser::Message{
+      location, parser::MessageFormattedText{fixedText, arg.data()}};
+}
+static parser::Message Error(const SourceName &location,
+    parser::MessageFixedText fixedText, const std::string &arg1,
+    const std::string &arg2) {
+  return parser::Message{location,
+      parser::MessageFormattedText{fixedText, arg1.data(), arg2.data()}};
+}
+
 }  // namespace Fortran::semantics
diff --git a/flang/lib/semantics/mod-file.h b/flang/lib/semantics/mod-file.h
index d0dcaa2..8d9c2d7 100644
--- a/flang/lib/semantics/mod-file.h
+++ b/flang/lib/semantics/mod-file.h
@@ -15,10 +15,43 @@
 #ifndef FORTRAN_SEMANTICS_MOD_FILE_H_
 #define FORTRAN_SEMANTICS_MOD_FILE_H_
 
+#include "resolve-names.h"
+#include "../parser/char-block.h"
+#include "../parser/message.h"
+#include "../parser/parse-tree.h"
+#include "../parser/parsing.h"
+#include "../parser/provenance.h"
+#include <iostream>
+#include <string>
+
 namespace Fortran::semantics {
 
+using SourceName = parser::CharBlock;
+
 void WriteModFiles();
 
+class ModFileReader {
+public:
+  // directories specifies where to search for module files
+  ModFileReader(const std::vector<std::string> &directories)
+    : directories_{directories} {}
+
+  // Find and read the module file for modName.
+  // Return true on success; otherwise errors() reports the problems.
+  bool Read(const SourceName &modName);
+  std::list<parser::Message> &errors() { return errors_; }
+
+private:
+  std::vector<std::string> directories_;
+  parser::AllSources allSources_;
+  std::unique_ptr<parser::CookedSource> cooked_{
+      std::make_unique<parser::CookedSource>(allSources_)};
+  std::list<parser::Message> errors_;
+
+  std::optional<std::string> FindModFile(const SourceName &);
+  bool Prescan(const SourceName &, const std::string &);
+};
+
 }  // namespace Fortran::semantics
 
 #endif
diff --git a/flang/lib/semantics/resolve-names.cc b/flang/lib/semantics/resolve-names.cc
index c91ed11..f0a6d02 100644
--- a/flang/lib/semantics/resolve-names.cc
+++ b/flang/lib/semantics/resolve-names.cc
@@ -27,6 +27,7 @@
 #include <ostream>
 #include <set>
 #include <stack>
+#include <vector>
 
 namespace Fortran::semantics {
 
@@ -387,6 +388,10 @@ public:
   bool Pre(const parser::UseStmt &);
   void Post(const parser::UseStmt &);
 
+  void add_searchDirectory(const std::string &dir) {
+    searchDirectories_.push_back(dir);
+  }
+
 private:
   // The default access spec for this module.
   Attr defaultAccess_{Attr::PUBLIC};
@@ -394,9 +399,12 @@ private:
   const SourceName *prevAccessStmt_{nullptr};
   // The scope of the module during a UseStmt
   const Scope *useModuleScope_{nullptr};
+  // Directories to search for .mod files
+  std::vector<std::string> searchDirectories_;
+
   void SetAccess(const parser::Name &, Attr);
   void ApplyDefaultAccess();
-
+  const Scope *FindModule(const SourceName &);
   void AddUse(const parser::Rename::Names &);
   void AddUse(const parser::Name &);
   // Record a use from useModuleScope_ of useName as localName. location is
@@ -1223,20 +1231,8 @@ bool ModuleVisitor::Pre(const parser::Rename::Names &x) {
 
 // Set useModuleScope_ to the Scope of the module being used.
 bool ModuleVisitor::Pre(const parser::UseStmt &x) {
-  // x.nature = UseStmt::ModuleNature::Intrinsic or Non_Intrinsic
-  const auto it{Scope::globalScope.find(x.moduleName.source)};
-  if (it == Scope::globalScope.end()) {
-    Say(x.moduleName, "Module '%s' not found"_err_en_US);
-    return false;
-  }
-  const auto *details{it->second->detailsIf<ModuleDetails>()};
-  if (!details) {
-    Say(x.moduleName, "'%s' is not a module"_err_en_US);
-    return false;
-  }
-  useModuleScope_ = details->scope();
-  CHECK(useModuleScope_);
-  return true;
+  useModuleScope_ = FindModule(x.moduleName.source);
+  return useModuleScope_ != nullptr;
 }
 void ModuleVisitor::Post(const parser::UseStmt &x) {
   if (const auto *list{std::get_if<std::list<parser::Rename>>(&x.u)}) {
@@ -1270,6 +1266,30 @@ void ModuleVisitor::Post(const parser::UseStmt &x) {
   useModuleScope_ = nullptr;
 }
 
+// Find the module with this name and return its scope.
+// May have to read a .mod file to find it.
+// Return nullptr on error, after reporting it.
+const Scope *ModuleVisitor::FindModule(const SourceName &name) {
+  auto it{Scope::globalScope.find(name)};
+  if (it == Scope::globalScope.end()) {
+    ModFileReader reader{searchDirectories_};
+    if (!reader.Read(name)) {
+      for (auto &error : reader.errors()) {
+        Say(std::move(error));
+      }
+      return nullptr;
+    }
+    it = Scope::globalScope.find(name);
+    CHECK(it != Scope::globalScope.end());  // else would have reported error
+  }
+  const auto *details{it->second->detailsIf<ModuleDetails>()};
+  if (!details) {
+    Say(name, "'%s' is not a module"_err_en_US);
+    return nullptr;
+  }
+  return details->scope();
+}
+
 void ModuleVisitor::AddUse(const parser::Rename::Names &names) {
   const SourceName &useName{std::get<0>(names.t).source};
   const SourceName &localName{std::get<1>(names.t).source};
@@ -1468,7 +1488,7 @@ void InterfaceVisitor::Post(const parser::GenericStmt &x) {
 void InterfaceVisitor::AddToGeneric(
     const parser::Name &name, bool expectModuleProc) {
   genericSymbol_->get<GenericDetails>().add_specificProcName(
-          name.source, expectModuleProc);
+      name.source, expectModuleProc);
 }
 void InterfaceVisitor::AddToGeneric(const Symbol &symbol) {
   genericSymbol_->get<GenericDetails>().add_specificProc(&symbol);
@@ -2421,9 +2441,13 @@ void ResolveNamesVisitor::Post(const parser::Program &) {
   CHECK(!GetDeclTypeSpec());
 }
 
-void ResolveNames(
-    parser::Program &program, const parser::CookedSource &cookedSource) {
+void ResolveNames(parser::Program &program,
+    const parser::CookedSource &cookedSource,
+    const std::vector<std::string> &searchDirectories) {
   ResolveNamesVisitor visitor;
+  for (auto &dir : searchDirectories) {
+    visitor.add_searchDirectory(dir);
+  }
   parser::Walk(const_cast<const parser::Program &>(program), visitor);
   if (!visitor.messages().empty()) {
     visitor.messages().Emit(std::cerr, cookedSource);
diff --git a/flang/lib/semantics/resolve-names.h b/flang/lib/semantics/resolve-names.h
index 5f52954..8a786a8c 100644
--- a/flang/lib/semantics/resolve-names.h
+++ b/flang/lib/semantics/resolve-names.h
@@ -16,6 +16,7 @@
 #define FORTRAN_SEMANTICS_RESOLVE_NAMES_H_
 
 #include <iosfwd>
+#include <vector>
 
 namespace Fortran::parser {
 struct Program;
@@ -24,7 +25,8 @@ class CookedSource;
 
 namespace Fortran::semantics {
 
-void ResolveNames(parser::Program &, const parser::CookedSource &);
+void ResolveNames(parser::Program &, const parser::CookedSource &,
+    const std::vector<std::string> &);
 void DumpSymbols(std::ostream &);
 
 }  // namespace Fortran::semantics
diff --git a/flang/lib/semantics/scope.cc b/flang/lib/semantics/scope.cc
index 22bea69..63e6d47 100644
--- a/flang/lib/semantics/scope.cc
+++ b/flang/lib/semantics/scope.cc
@@ -60,8 +60,8 @@ std::ostream &operator<<(std::ostream &os, const Scope &scope) {
   }
   os << scope.children_.size() << " children\n";
   for (const auto &pair : scope.symbols_) {
-    const auto &symbol{pair.second};
-    os << "  " << symbol << '\n';
+    const auto *symbol{pair.second};
+    os << "  " << *symbol << '\n';
   }
   return os;
 }
diff --git a/flang/lib/semantics/scope.h b/flang/lib/semantics/scope.h
index 9215720..544435f 100644
--- a/flang/lib/semantics/scope.h
+++ b/flang/lib/semantics/scope.h
@@ -106,6 +106,11 @@ public:
 
   DerivedTypeSpec &MakeDerivedTypeSpec(const SourceName &);
 
+  std::unique_ptr<parser::CookedSource> cooked_;
+  void set_cookedSource(std::unique_ptr<parser::CookedSource> cooked) {
+    cooked_ = std::move(cooked);
+  }
+
 private:
   Scope &parent_;
   const Kind kind_;
diff --git a/flang/lib/semantics/symbol.h b/flang/lib/semantics/symbol.h
index f14c723..0bdc5b3 100644
--- a/flang/lib/semantics/symbol.h
+++ b/flang/lib/semantics/symbol.h
@@ -233,7 +233,7 @@ std::string DetailsToString(const Details &);
 
 class Symbol {
 public:
-  ENUM_CLASS(Flag, Function, Subroutine, Implicit);
+  ENUM_CLASS(Flag, Function, Subroutine, Implicit, ModFile);
   using Flags = common::EnumSet<Flag, Flag_enumSize>;
 
   const Scope &owner() const { return *owner_; }
diff --git a/flang/test/semantics/resolve12.f90 b/flang/test/semantics/resolve12.f90
index fd82e87..a3f38d2 100644
--- a/flang/test/semantics/resolve12.f90
+++ b/flang/test/semantics/resolve12.f90
@@ -19,7 +19,7 @@ subroutine sub
 end
 
 use m1
-!ERROR: Module 'm2' not found
+!ERROR: Cannot find module file for 'm2'
 use m2
 !ERROR: 'sub' is not a module
 use sub
diff --git a/flang/test/semantics/test_errors.sh b/flang/test/semantics/test_errors.sh
index 46a3a3a..fd08629 100755
--- a/flang/test/semantics/test_errors.sh
+++ b/flang/test/semantics/test_errors.sh
@@ -18,7 +18,7 @@
 
 PATH=/usr/bin:/bin
 srcdir=$(dirname $0)
-CMD="${F18:-../../tools/f18/f18} -fdebug-resolve-names -fparse-only"
+CMD="${F18:-../../../tools/f18/f18} -fdebug-resolve-names -fparse-only"
 
 if [[ $# != 1 ]]; then
   echo "Usage: $0 <fortran-source>"
@@ -38,7 +38,7 @@ expect=$temp/expect
 diffs=$temp/diffs
 
 cmd="$CMD $src"
-$cmd > $log 2>&1
+( cd $temp; $cmd ) > $log 2>&1
 [[ $? -ge 128 ]] && exit 1
 
 # $actual has errors from the compiler; $expect has them from !ERROR comments in source
diff --git a/flang/tools/f18/f18.cc b/flang/tools/f18/f18.cc
index f1771d6..3288637 100644
--- a/flang/tools/f18/f18.cc
+++ b/flang/tools/f18/f18.cc
@@ -206,7 +206,9 @@ std::string CompileFortran(
   }
   if (driver.debugResolveNames || driver.dumpSymbols ||
       driver.dumpUnparseWithSymbols) {
-    Fortran::semantics::ResolveNames(parseTree, parsing.cooked());
+    std::vector<std::string> directories{options.searchDirectories};
+    directories.insert(directories.begin(), "."s);
+    Fortran::semantics::ResolveNames(parseTree, parsing.cooked(), directories);
     Fortran::semantics::WriteModFiles();
     if (driver.dumpSymbols) {
       Fortran::semantics::DumpSymbols(std::cout);
@@ -452,7 +454,8 @@ int main(int argc, char *const argv[]) {
   if (options.isStrictlyStandard) {
     options.features.WarnOnAllNonstandard();
   }
-  if (!options.features.IsEnabled(Fortran::parser::LanguageFeature::BackslashEscapes)) {
+  if (!options.features.IsEnabled(
+          Fortran::parser::LanguageFeature::BackslashEscapes)) {
     driver.pgf90Args.push_back("-Mbackslash");
   }