From 10fceb342f592b17fcdff717ffaa146f20fe8ef3 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EC=A1=B0=EC=9B=85=EC=84=9D/Common=20Platform=20Lab=28SR=29?= =?utf8?q?/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Tue, 28 Jun 2022 09:01:54 +0900 Subject: [PATCH] Renew InputBubble Option (#416) When the native image compiled with the --inputbubble option is executed, all assemblies included in the same bubble must be compiled too. Previous --ni-dll or --no-pipeline option can lead misusage of --inputbubble option. For example, if the below command are executed, system libraries are compiled one by one. $ dotnettool --inputbubble --no-pipeline --ni-dir /usr/share/dotnet.tizen/netcoreapp In this case, a problem may occur because crossgen2 is executed with compiled native images with --inputbubble and the non-compiled assemblies In order to prevent errors caused by such misuse, the following modifications were made. 1. When using no-pipeline mode, it was temporarily created as ni.dll.tmp and then changed to ni.dll in the last step of dotnettool. 2. if --inputbubbleref option is not set, only input file(s) of dotnettool is added to bubble Additionally, for fine tunning, --ref and --inputbubbleref option get file path(s) ot directory path(s) --- NativeLauncher/inc/ni_common.h | 8 +- NativeLauncher/tool/dotnettool.cc | 60 ++++++----- NativeLauncher/tool/ni_common.cc | 212 +++++++++++++++++++++++--------------- NativeLauncher/tool/nitool.cc | 2 +- tests/TCs/6_TOOL/TOOL.py | 12 +-- 5 files changed, 172 insertions(+), 122 deletions(-) diff --git a/NativeLauncher/inc/ni_common.h b/NativeLauncher/inc/ni_common.h index cec0125..22f69e4 100644 --- a/NativeLauncher/inc/ni_common.h +++ b/NativeLauncher/inc/ni_common.h @@ -31,7 +31,6 @@ #define NI_FLAGS_APPNI 0x0001 #define NI_FLAGS_VERBOSE 0x0002 #define NI_FLAGS_APP_UNDER_RO_AREA 0x0004 -#define NI_FLAGS_REF 0x0008 #define NI_FLAGS_INPUT_BUBBLE 0x0010 #define NI_FLAGS_INPUT_BUBBLE_REF 0x0020 #define NI_FLAGS_EXTRA_REF 0x0040 @@ -56,9 +55,9 @@ typedef enum { typedef struct NIOption{ DWORD flags; - std::vector refPath; + std::vector refFiles; std::vector extraRefPath; - std::vector inputBubbleRefPath; + std::vector inputBubbleRefFiles; std::vector mibcPath; } NIOption; @@ -82,10 +81,11 @@ void finalizeNICommon(); /** * @brief create native images for platform DLLs (.NETCore + TizenFX) + * @param[in] paths paths to directories which contains extra platform dlls * @param[in] flags additional flags for the image generator * @return ni_error_e */ -ni_error_e createNIPlatform(NIOption* opt); +ni_error_e createNIPlatform(std::string& extraInputs, NIOption* opt); /** * @brief create a native image for a single DLL diff --git a/NativeLauncher/tool/dotnettool.cc b/NativeLauncher/tool/dotnettool.cc index 3c15704..e145699 100644 --- a/NativeLauncher/tool/dotnettool.cc +++ b/NativeLauncher/tool/dotnettool.cc @@ -57,11 +57,14 @@ void DisplayUsage() { "Options:\n" " --mibc - Specify Mibc files. Sepatated with ':'.\n" " --verbose - Display verbose information\n" - " --inputbubble - Use inputbubble\n" - " --inputbubbleref - Path of references for inputbubble (used with --inputbubble option)\n" - " (If not set, all references are included to inputbubble.)\n" - " --ref - Path of references\n" - " (If not set, default system paths are used.)\n" + " --inputbubble - Compile input assemblies into one bubble with the assemblies described at --inputbubbleref option\n" + " Note!: If you do not have an accurate understanding of Bubble, do not use this option.\n" + " All assemblies in the bubble must be guaranteed to be compiled to the native image before execution.\n" + " If an uncompiled assembly is included in the bubble during execution, an unknown error may occur.\n" + " If --inputbubbleref option doesnot be set, only input files are included to bubble. \n" + " --inputbubbleref - Input bubble reference file(s) to be added to bubble (used with --inputbubble option)\n" + " --ref - Reference file(s) for compilation\n" + " (system paths are set automatically.)\n" " --no-pipeline - Compile the dlls individually\n" " --print-cmd - Print command and options\n" " --skip-ro-app - Skip re-generate NI for apps installed RO area\n" @@ -153,14 +156,10 @@ int main(int argc, char* argv[]) opt->flags |= NI_FLAGS_INPUT_BUBBLE_REF; - std::vector paths; - splitPath(std::string(argv[i]), paths); - for (const auto &path : paths) { - if (!isDirectory(path)) { - _SERR("intpubbubbleref path is missing or not exist"); - return -1; - } - opt->inputBubbleRefPath.push_back(path); + std::vector files; + splitPath(std::string(argv[i]), files); + for (const auto &f : files) { + opt->inputBubbleRefFiles.push_back(f); } } else if (arg == "--ref") { ++i; @@ -170,16 +169,10 @@ int main(int argc, char* argv[]) return 0; } - opt->flags |= NI_FLAGS_REF; - - std::vector paths; - splitPath(std::string(argv[i]), paths); - for (const auto &path : paths) { - if (!isDirectory(path)) { - _SERR("ref path is missing or not exist"); - return -1; - } - opt->refPath.push_back(path); + std::vector files; + splitPath(std::string(argv[i]), files); + for (const auto &f : files) { + opt->refFiles.push_back(f); } } else { args.push_back(arg); @@ -200,9 +193,15 @@ int main(int argc, char* argv[]) std::string cmd = std::string(*it); it = args.erase(it); - //sh-3.2# dotnettool --ni-system + //sh-3.2# dotnettool --ni-system [AssemblyDirectory] [AssemblyDirectory] ... if (cmd == "--ni-system") { - int ret = createNIPlatform(opt); + std::string inputs; + while (it != args.end()) { + const std::string dir = std::string(*it); + inputs = inputs + ":" + dir; + it = args.erase(it); + } + int ret = createNIPlatform(inputs, opt); if (ret != NI_ERROR_NONE) { _SERR("Failed to generate system NI"); } @@ -212,14 +211,23 @@ int main(int argc, char* argv[]) if (args.size() < 1) { _SERR("DLL path is missing"); } + std::vector inputs; while (it != args.end()) { std::string dll = std::string(*it); + inputs.push_back(dll); + opt->refFiles.push_back(dll); + if (opt->flags & NI_FLAGS_INPUT_BUBBLE) { + opt->inputBubbleRefFiles.push_back(dll); + } + it = args.erase(it); + } + + for (auto &dll : inputs) { int ret = createNIDll(dll, opt); if (ret != NI_ERROR_NONE && ret != NI_ERROR_ALREADY_EXIST) { _SERR("Failed to generate NI file [%s]", dll.c_str()); break; } - it = args.erase(it); } } //sh-3.2# dotnettool --ni-pkg [pkgId] [pkgId] ... diff --git a/NativeLauncher/tool/ni_common.cc b/NativeLauncher/tool/ni_common.cc index a5e3c83..ea63ecf 100644 --- a/NativeLauncher/tool/ni_common.cc +++ b/NativeLauncher/tool/ni_common.cc @@ -385,10 +385,7 @@ static void makeArgs(std::vector& args, const std::vectorflags & NI_FLAGS_NO_PIPELINE)) { - args.push_back(CROSSGEN_OPT_OUT_NEAR_INPUT); - args.push_back(CROSSGEN_OPT_SINGLE_FILE_COMPILATION); - } + //args.push_back(OPT_PARALLELISM); //args.push_back(OPT_PARALLELISM_COUNT); args.push_back(CROSSGEN_OPT_RESILIENT); @@ -399,21 +396,12 @@ static void makeArgs(std::vector& args, const std::vectorflags & NI_FLAGS_INPUT_BUBBLE_REF) { - INPUTBUBBLE_REF_VECTOR.clear(); - // check inputbubbleref format. - for (const auto &path : opt->inputBubbleRefPath) { - if (checkDllExistInDir(path)) { - INPUTBUBBLE_REF_VECTOR.push_back("--inputbubbleref:" + path + "/*.dll"); - } - } - // add ref path to inputbubble ref - for (const auto &path : refPaths) { - if (checkDllExistInDir(path)) { - INPUTBUBBLE_REF_VECTOR.push_back("--inputbubbleref:" + path + "/*.dll"); - } - } - for (const auto &path : INPUTBUBBLE_REF_VECTOR) { + INPUTBUBBLE_REF_VECTOR.clear(); + for (const auto &path : opt->inputBubbleRefFiles) { + INPUTBUBBLE_REF_VECTOR.push_back("--inputbubbleref:" + path); + } + for (const auto &path : INPUTBUBBLE_REF_VECTOR) { + if (find(args.begin(), args.end(), path) == args.end()) { args.push_back(path.c_str()); } } @@ -425,7 +413,9 @@ static void makeArgs(std::vector& args, const std::vector& args, const std::vectorflags & NI_FLAGS_REF) { - for (const auto &path : opt->refPath) { + for (const auto &path : opt->refFiles) { + REF_VECTOR.push_back("-r:" + path); + } + + std::vector paths = __pm->getPlatformAssembliesPaths(); + for (const auto &path : paths) { + if (checkDllExistInDir(path)) { REF_VECTOR.push_back("-r:" + path + "/*.dll"); } - } else { - std::vector paths = __pm->getPlatformAssembliesPaths(); - for (const auto &path : paths) { - if (checkDllExistInDir(path)) { - REF_VECTOR.push_back("-r:" + path + "/*.dll"); - } - } } if (opt->flags & NI_FLAGS_EXTRA_REF) { @@ -465,7 +453,9 @@ static void makeArgs(std::vector& args, const std::vectorflags & NI_FLAGS_APPNI) { - std::string appNIPath = getAppNIFilePath(dllPath, opt); - moveFile(niPath, appNIPath); - makePdbSymlinkForNI(dllPath, appNIPath); - _SOUT("Native image %s generated successfully.", appNIPath.c_str()); - } else { - _SOUT("Native image %s generated successfully.", niPath.c_str()); + outFile = getAppNIFilePath(dllPath, opt); + makePdbSymlinkForNI(dllPath, outFile); + + if (opt->flags & NI_FLAGS_INPUT_BUBBLE && opt->flags & NI_FLAGS_NO_PIPELINE) { + outFile = outFile + ".tmp"; + } + + if (niPath != outFile) { + moveFile(niPath, outFile); + } + } + + if (!(opt->flags & NI_FLAGS_INPUT_BUBBLE && opt->flags & NI_FLAGS_NO_PIPELINE)) { + _SOUT("Native image %s generated successfully.", outFile.c_str()); } + return NI_ERROR_NONE; } @@ -537,6 +536,8 @@ static ni_error_e crossgen2PipeLine(const std::vector& dllList, con } else { std::vector argv; makeArgs(argv, refPaths, opt); + argv.push_back(CROSSGEN_OPT_OUT_NEAR_INPUT); + argv.push_back(CROSSGEN_OPT_SINGLE_FILE_COMPILATION); // add input files at the end of parameter for (const auto &input : dllList) { @@ -571,13 +572,15 @@ static ni_error_e crossgen2NoPipeLine(const std::vector& dllList, c } else { niPath = getNIFilePath(dllPath); } - #ifdef UNIQUE_DEFAULT_BASE_ADDR_SUPPORT uintptr_t baseAddr = 0; if (isTPADll(dllPath)) { baseAddr = getNextBaseAddr(); } #endif + if (opt->flags & NI_FLAGS_INPUT_BUBBLE) { + niPath += ".tmp"; + } // fork crossgen2 pid_t pid = fork(); @@ -648,6 +651,7 @@ static ni_error_e createCoreLibNI(NIOption* opt) { std::string coreLib = concatPath(__pm->getRuntimePath(), "System.Private.CoreLib.dll"); std::string niCoreLib = concatPath(__pm->getRuntimePath(), "System.Private.CoreLib.ni.dll"); + std::string niTmpCoreLib = concatPath(__pm->getRuntimePath(), "System.Private.CoreLib.ni.dll.tmp"); std::string coreLibBackup = concatPath(__pm->getRuntimePath(), "System.Private.CoreLib.dll.Backup"); std::vector dllList; @@ -655,14 +659,21 @@ static ni_error_e createCoreLibNI(NIOption* opt) dllList.push_back(getAbsolutePath(coreLib)); if (!isFile(coreLibBackup) && !isR2RImage(coreLib)) { - if (crossgen2NoPipeLine(dllList, refPaths, opt) == NI_ERROR_NONE && exist(niCoreLib)) { + if (crossgen2NoPipeLine(dllList, refPaths, opt) == NI_ERROR_NONE) { if (rename(coreLib.c_str(), coreLibBackup.c_str())) { - _SERR("Failed to rename System.Private.CoreLib.dll"); + _SERR("Failed to rename from System.Private.CoreLib.dll to System.Private.CoreLib.dll.Backup"); return NI_ERROR_CORE_NI_FILE; } - if (rename(niCoreLib.c_str(), coreLib.c_str())) { - _SERR("Failed to rename System.Private.CoreLib.ni.dll"); - return NI_ERROR_CORE_NI_FILE; + if (opt->flags & NI_FLAGS_INPUT_BUBBLE) { + if (rename(niTmpCoreLib.c_str(), coreLib.c_str())) { + _SERR("Failed to rename from System.Private.CoreLib.ni.dll.tmp to Private.CoreLib.dll"); + return NI_ERROR_CORE_NI_FILE; + } + } else { + if (rename(niCoreLib.c_str(), coreLib.c_str())) { + _SERR("Failed to rename from System.Private.CoreLib.ni.dll to Private.CoreLib.dll"); + return NI_ERROR_CORE_NI_FILE; + } } } else { _SERR("Failed to create native image for %s", coreLib.c_str()); @@ -731,33 +742,39 @@ static ni_error_e doAOTList(std::vector& dllList, const std::string if (opt->flags & NI_FLAGS_NO_PIPELINE) { ret = crossgen2NoPipeLine(dllList, paths, opt); } else { - // When the forked process in the pipeline state is terminated(WIFSIGNALED(status)), - // retry the generation of the native image - // if the number of .dll files and the number of .ni.dll files are different. - for (int callCnt = 0; callCnt < 2; callCnt++) { - // If an error occurs, perform it twice with the same option. - ret = crossgen2PipeLine(dllList, paths, opt); - if (ret != NI_ERROR_NONE) { - _SERR("Crossgen2 is abnormally terminated. Regenerate native images that failed while running crossgen2."); - dllList.clear(); - for (auto it = niList.begin(); it != niList.end(); it++) { - std::string niPath = *it; - std::string dllPath = changeExtension(niPath, ".ni.dll", ".dll"); - if (crossgen2PostAction(dllPath, niPath, opt) != NI_ERROR_NONE) { - dllList.push_back(dllPath); - } else { - niList.erase(it--); - } + std::vector notCompiled; + ret = crossgen2PipeLine(dllList, paths, opt); + if (ret != NI_ERROR_NONE) { + _SERR("Crossgen2 is abnormally terminated. Regenerate native images that failed while running crossgen2."); + for (auto &dll : dllList) { + std::string tFile = changeExtension(dll, ".dll", ".ni.dll"); + if (opt->flags & NI_FLAGS_INPUT_BUBBLE) { + tFile += ".tmp"; + } + if (!exist(tFile)) { + notCompiled.push_back(dll); } - } else { - break; } - } - // If an error occurs after two crossgen2PipeLine() attempts, - // try crossgen2NoPipeLine() for the last time. - if (ret != NI_ERROR_NONE) { _SERR("Retry running crossgen2 with --no-pipeline mode to avoid termination by OOM."); - ret = crossgen2NoPipeLine(dllList, paths, opt); + ret = crossgen2NoPipeLine(notCompiled, paths, opt); + } + } + + if (ret == NI_ERROR_NONE && opt->flags & NI_FLAGS_INPUT_BUBBLE) { + for (auto &dll : dllList) { + std::string tmpFile; + std::string niFile; + if (opt->flags & NI_FLAGS_APPNI) { + niFile = getAppNIFilePath(dll, opt); + } else { + niFile = getNIFilePath(dll); + } + tmpFile = niFile + ".tmp"; + + if (exist(tmpFile)) { + moveFile(tmpFile, niFile); + _SOUT("Native image %s generated successfully.", niFile.c_str()); + } } } @@ -890,14 +907,12 @@ void finalizeNICommon() } } -ni_error_e createNIPlatform(NIOption* opt) +ni_error_e createNIPlatform(std::string& extraInputs, NIOption* opt) { - ni_error_e ret = createNIUnderDirs(__pm->getRuntimePath(), opt); - if (ret != NI_ERROR_NONE) { - return ret; - } + extraInputs += ":" + __pm->getRuntimePath(); + extraInputs += ":" + __pm->getTizenFXPath(); - return createNIUnderDirs(__pm->getTizenFXPath(), opt); + return createNIUnderDirs(extraInputs, opt); } ni_error_e createNIDll(const std::string& dllPath, NIOption* opt) @@ -909,6 +924,21 @@ ni_error_e createNIUnderTAC(const std::string& targetPath, const std::string& re { ni_error_e ret; + bool isAppNI = false; + if (opt->flags & NI_FLAGS_APPNI) { + isAppNI = true; + } + + if (opt->flags & NI_FLAGS_INPUT_BUBBLE) { + std::vector refs; + splitPath(refPaths, refs); + for (auto &p: refs) { + if (isDirectory(p) && checkDllExistInDir(p)) { + opt->inputBubbleRefFiles.push_back(p + "/*.dll"); + } + } + } + // get managed file list from targetPath std::vector dllList; ret = getTargetDllList(targetPath, dllList); @@ -931,20 +961,24 @@ ni_error_e createNIUnderTAC(const std::string& targetPath, const std::string& re // So, unset NI_FLAGS_APPNI temporally and restore it after running AOT. opt->flags &= ~NI_FLAGS_APPNI; ret = doAOTList(needNIList, refPaths, opt); - opt->flags |= NI_FLAGS_APPNI; + if (isAppNI) { + opt->flags |= NI_FLAGS_APPNI; + } if (ret != NI_ERROR_NONE) { return ret; } } - for (auto &niPath : niList) { - if (exist(niPath)) { - std::string symNIPath = concatPath(targetPath, getFileName(niPath)); - if (!exist(symNIPath)) { - bf::create_symlink(niPath, symNIPath); - copySmackAndOwnership(targetPath.c_str(), symNIPath.c_str(), true); - _SOUT("%s symbolic link file generated successfully.", symNIPath.c_str()); - _INFO("%s symbolic link file generated successfully.", symNIPath.c_str()); + if (isAppNI) { + for (auto &niPath : niList) { + if (exist(niPath)) { + std::string symNIPath = concatPath(targetPath, getFileName(niPath)); + if (!exist(symNIPath)) { + bf::create_symlink(niPath, symNIPath); + copySmackAndOwnership(targetPath.c_str(), symNIPath.c_str(), true); + _SOUT("%s symbolic link file generated successfully.", symNIPath.c_str()); + _INFO("%s symbolic link file generated successfully.", symNIPath.c_str()); + } } } } @@ -961,6 +995,14 @@ ni_error_e createNIUnderDirs(const std::string& rootPaths, NIOption* opt) std::vector paths; splitPath(rootPaths, paths); + if (opt->flags & NI_FLAGS_INPUT_BUBBLE) { + for (auto &p: paths) { + if (isDirectory(p) && checkDllExistInDir(p)) { + opt->inputBubbleRefFiles.push_back(p + "/*.dll"); + } + } + } + for (const auto &path : paths) { if (!exist(path)) { continue; diff --git a/NativeLauncher/tool/nitool.cc b/NativeLauncher/tool/nitool.cc index b48c076..943852e 100644 --- a/NativeLauncher/tool/nitool.cc +++ b/NativeLauncher/tool/nitool.cc @@ -73,7 +73,7 @@ int main(int argc, char* argv[]) help(argv[0]); return 0; } else if (cmdOptionExists(argv, argv+argc, "--system")) { - createNIPlatform(opt->flags); + createNIPlatform("", opt->flags); return 0; } else if (cmdOptionExists(argv, argv+argc, "--dll")) { dllMode = true; diff --git a/tests/TCs/6_TOOL/TOOL.py b/tests/TCs/6_TOOL/TOOL.py index e331316..d6bcd45 100755 --- a/tests/TCs/6_TOOL/TOOL.py +++ b/tests/TCs/6_TOOL/TOOL.py @@ -560,7 +560,7 @@ def TC_20(): ("+ --out-near-input" not in raw) or \ ("+ --single-file-compilation" not in raw) or \ ("+ --resilient" not in raw) or \ - ("+ -O" not in raw) or \ + ("+ --Ot" not in raw) or \ (f"+ {FRAMEWORK_DIR}Tizen.dll" not in raw) or \ ("+ (null)" not in raw): return "FAIL : Print command and option while creating native image for Tizen.dll" @@ -570,7 +570,7 @@ def TC_20(): # Create native image for netstandard.dll by adding options --inputbubble and --compilebubblegenerics. def TC_21(): cmd(f"shell mount -o remount,rw /") - raw = cmd(f"shell dotnettool --ni-dll --inputbubble --print-cmd {RUNTIME_DIR}netstandard.dll") + raw = cmd(f"shell dotnettool --inputbubble --inputbubbleref \'{RUNTIME_DIR}*.dll\' --print-cmd --ni-dll {RUNTIME_DIR}netstandard.dll") if ("netstandard.ni.dll generated successfully." not in raw) or \ ("+ --inputbubble" not in raw) or \ ("+ --compilebubblegenerics" not in raw): @@ -581,7 +581,7 @@ def TC_21(): # Create native image for System.dll by adding options --inputbubble and --inputbubbleref. def TC_22(): cmd(f"shell mount -o remount,rw /") - raw = cmd(f"shell dotnettool --ni-dll --inputbubble --inputbubbleref {RUNTIME_DIR}crossgen2 --print-cmd {RUNTIME_DIR}System.dll") + raw = cmd(f"shell dotnettool --inputbubble --inputbubbleref \'{RUNTIME_DIR}crossgen2/*.dll\' --print-cmd --ni-dll {RUNTIME_DIR}System.dll") if ("System.ni.dll generated successfully." not in raw) or \ ("+ --inputbubble" not in raw) or \ ("+ --inputbubbleref:/usr/share/dotnet.tizen/netcoreapp/crossgen2/*.dll" not in raw): @@ -592,10 +592,10 @@ def TC_22(): # Create native image for System.Console.dll by adding option --ref. def TC_23(): cmd(f"shell mount -o remount,rw /") - raw = cmd(f"shell dotnettool --ni-dll --ref {RUNTIME_DIR}:{RUNTIME_DIR}crossgen2 --print-cmd {RUNTIME_DIR}System.Console.dll") + raw = cmd(f"shell dotnettool --ref \'{RUNTIME_DIR}*.dll:{RUNTIME_DIR}crossgen2/*.dll\' --print-cmd --ni-dll {RUNTIME_DIR}System.Console.dll") if ("System.Console.ni.dll generated successfully." not in raw) or \ - (f"+ -r:{RUNTIME_DIR}/*.dll" not in raw) or \ - (f"+ -r:{RUNTIME_DIR}crossgen2/*.dll" not in raw): + (f"+ -r:/usr/share/dotnet.tizen/netcoreapp/*.dll" not in raw) or \ + (f"+ -r:/usr/share/dotnet.tizen/netcoreapp/crossgen2/*.dll" not in raw): return "FAIL : Create native image for System.Console.dll by adding option --ref" return "PASS" -- 2.7.4