Forked process in pipeline state can be terminated while crossgen2 is running

Change-Id: Ib3a1851a7f43fedd270519c6389482c6ca9568a5

diff --git a/NativeLauncher/inc/ni_common.h b/NativeLauncher/inc/ni_common.h
index 8ce0346..349e3dc 100644 (file)
--- a/NativeLauncher/inc/ni_common.h
+++ b/NativeLauncher/inc/ni_common.h
@@ -48,6 +48,7 @@ typedef enum {
        NI_ERROR_INVALID_PACKAGE = -4,
        NI_ERROR_NOT_SUPPORTED = -5,
        NI_ERROR_CORE_NI_FILE = -6,
+       NI_ERROR_ABNORMAL_PROCESS_TERMINATION = -7,
        NI_ERROR_UNKNOWN = -9
 } ni_error_e;
 

diff --git a/NativeLauncher/tool/ni_common.cc b/NativeLauncher/tool/ni_common.cc
index 7c146b7..402e34a 100644 (file)
--- a/NativeLauncher/tool/ni_common.cc
+++ b/NativeLauncher/tool/ni_common.cc
@@ -396,6 +396,25 @@ static ni_error_e makePdbSymlinkForNI(std::string dllPath, std::string niPath)
        return NI_ERROR_NONE;
 }
 
+static ni_error_e crossgen2PostAction(const std::string& dllPath, const std::string& niPath, NIOption* opt) {
+       if (!exist(niPath)) {
+               removeFile(changeExtension(niPath, ".ni.dll", ".ni.dll.tmp"));
+               _SERR("Fail to create native image for %s", dllPath.c_str());
+               return NI_ERROR_NO_SUCH_FILE;
+       }
+       copySmackAndOwnership(dllPath, niPath);
+       // if AppNI then move ni.dll file to .native_image and copy pdb to .native_image
+       if (opt->flags & NI_FLAGS_APPNI) {
+               std::string appNIPath = getAppNIFilePath(dllPath, opt);
+               moveFile(niPath, appNIPath);
+               makePdbSymlinkForNI(dllPath, niPath);
+               _SOUT("Native image %s generated successfully.", appNIPath.c_str());
+       } else {
+               _SOUT("Native image %s generated successfully.", niPath.c_str());
+       }
+       return NI_ERROR_NONE;
+}
+
 static ni_error_e crossgen2PipeLine(const std::vector<std::string>& dllList, const std::vector<std::string>& refPaths, NIOption* opt)
 {
        // fork crossgen2
@@ -408,26 +427,14 @@ static ni_error_e crossgen2PipeLine(const std::vector<std::string>& dllList, con
                waitpid(pid, &status, 0);
                if (WIFEXITED(status)) {
                        for (auto& dllPath: dllList) {
-                               std::string niPath = changeExtension(dllPath, ".dll", ".ni.dll");
-
-                               if (!exist(niPath)) {
-                                       _SERR("Fail to create native image for %s", dllPath.c_str());
-                                       return NI_ERROR_NO_SUCH_FILE;
-                               }
-
-                               copySmackAndOwnership(dllPath, niPath);
-                               // if AppNI then move ni.dll file to .native_image and copy pdb to .native_image
-                               if (opt->flags & NI_FLAGS_APPNI) {
-                                       std::string appNIPath = getAppNIFilePath(dllPath, opt);
-                                       moveFile(niPath, appNIPath);
-                                       makePdbSymlinkForNI(dllPath, appNIPath);
-                                       niPath = appNIPath;
+                               ni_error_e ret = crossgen2PostAction(dllPath, changeExtension(dllPath, ".dll", ".ni.dll"), opt);
+                               if (ret != NI_ERROR_NONE) {
+                                       return ret;
                                }
-
-                               _SOUT("Native image %s generated successfully.", niPath.c_str());
                        }
                } else {
                        _SERR("Failed. Forked process terminated abnormally");
+                       return NI_ERROR_ABNORMAL_PROCESS_TERMINATION;
                }
        } else {
                std::vector<const char*> argv;
@@ -476,19 +483,15 @@ static ni_error_e crossgen2NoPipeLine(const std::vector<std::string>& dllList, c
                        int status;
                        waitpid(pid, &status, 0);
                        if (WIFEXITED(status)) {
-                               if (!exist(niPath)) {
-                                       _SERR("Fail to create native image for %s", dllPath.c_str());
-                                       return NI_ERROR_NO_SUCH_FILE;
-                               }
-
-                               copySmackAndOwnership(dllPath, niPath);
-                               if (opt->flags & NI_FLAGS_APPNI) {
-                                       makePdbSymlinkForNI(dllPath, niPath);
+                               ni_error_e ret = crossgen2PostAction(dllPath, niPath, opt);
+                               if (ret != NI_ERROR_NONE) {
+                                       return ret;
                                }
-
-                               _SOUT("Native image %s generated successfully.", niPath.c_str());
                        } else {
                                _SERR("Failed. Forked process terminated abnormally");
+                               _SERR("Crossgen2 was terminated by the OOM killer. Please check the system.");
+                               removeFile(changeExtension(niPath, ".ni.dll", ".ni.dll.tmp"));
+                               return NI_ERROR_ABNORMAL_PROCESS_TERMINATION;
                        }
                } else {
                        std::vector<const char*> argv;
@@ -562,6 +565,7 @@ static ni_error_e doAOTList(std::vector<std::string>& dllList, const std::string
        std::string coreLib = concatPath(__pm->getRuntimePath(), "System.Private.CoreLib.dll");
        bool hasSPC = false;
 
+       std::vector<std::string> niList;
        for (auto it = dllList.begin(); it != dllList.end(); it++) {
                std::string f = *it;
                if (!isFile(f)) {
@@ -577,6 +581,8 @@ static ni_error_e doAOTList(std::vector<std::string>& dllList, const std::string
                if (f == coreLib) {
                        hasSPC = true;
                        dllList.erase(it--);
+               } else {
+                       niList.push_back(changeExtension(f, ".dll", ".ni.dll"));
                }
        }
 
@@ -605,7 +611,34 @@ static ni_error_e doAOTList(std::vector<std::string>& dllList, const std::string
        if (opt->flags & NI_FLAGS_NO_PIPELINE) {
                ret = crossgen2NoPipeLine(dllList, paths, opt);
        } else {
-               ret = crossgen2PipeLine(dllList, paths, opt);
+               // When the forked process in the pipeline state is terminated(WIFSIGNALED(status)),
+               // retry the generation of the native image
+               // if the number of .dll files and the number of .ni.dll files are different.
+               for (int callCnt = 0; callCnt < 2; callCnt++) {
+                       // If an error occurs, perform it twice with the same option.
+                       ret = crossgen2PipeLine(dllList, paths, opt);
+                       if (ret != NI_ERROR_NONE) {
+                               _SERR("Crossgen2 is abnormally terminated. Regenerate native images that failed while running crossgen2.");
+                               dllList.clear();
+                               for (auto it = niList.begin(); it != niList.end(); it++) {
+                                       std::string niPath = *it;
+                                       std::string dllPath = changeExtension(niPath, ".ni.dll", ".dll");
+                                       if (crossgen2PostAction(dllPath, niPath, opt) != NI_ERROR_NONE) {
+                                               dllList.push_back(dllPath);
+                                       } else {
+                                               niList.erase(it--);
+                                       }
+                               }
+                       } else {
+                               break;
+                       }
+               }
+               // If an error occurs after two crossgen2PipeLine() attempts,
+               // try crossgen2NoPipeLine() for the last time.
+               if (ret != NI_ERROR_NONE) {
+                       _SERR("Retry running crossgen2 with --no-pipeline mode to avoid termination by OOM.");
+                       ret = crossgen2NoPipeLine(dllList, paths, opt);
+               }
        }
 
        return ret;