From: Gleb Balykov Date: Thu, 23 Jun 2022 09:58:18 +0000 (+0300) Subject: [Tizen] Add truncate command to mcj-edit X-Git-Tag: accepted/tizen/unified/20221103.165808~16 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=3e036b7f9c06fe2b65dc308dcafff6e9f88249b7;p=platform%2Fupstream%2Fdotnet%2Fruntime.git [Tizen] Add truncate command to mcj-edit --- diff --git a/src/coreclr/tools/mcj-edit/README.md b/src/coreclr/tools/mcj-edit/README.md index 15b05b5..6a880ab 100644 --- a/src/coreclr/tools/mcj-edit/README.md +++ b/src/coreclr/tools/mcj-edit/README.md @@ -71,7 +71,7 @@ After this command two new files will be created: `pwd`/profile.dat.app for app- python3 mcj-edit.py merge -i `pwd`/profile.dat -i /tmp/profile2.dat -o `pwd`/profile.merged.dat ``` -After this command new file wil be created: `pwd`/profile.merged.dat. +After this command new file will be created: `pwd`/profile.merged.dat. **Note: merge can't be performed on two arbitrary mcj profiles! Next profiles can't be merged:** - if one profile contains module with name `AAA` and version `X` and another profile contains module with same name `AAA` and version `Y` @@ -79,12 +79,23 @@ After this command new file wil be created: `pwd`/profile.merged.dat. - if one profile contains module with name `AAA` with flags `X` and another profile contains module with same name `AAA` with flags `Y` (this situation should not happen now, flags are always 0) - if one profile contains method with token/signature `XXX` with flags `X` and another profile contains method with same token/signature `XXX` with flags `Y` (this situation can happen now only if one profile was rewritten during use, i.e. JIT_BY_APP_THREAD_TAG was not set for some methods, i.e. COMPlus_MultiCoreJitNoProfileGather=1 was not set) +### To truncate tail of profile + +```sh +python3 mcj-edit.py truncate --percent 50 -i `pwd`/profile.dat -i /tmp/profile2.dat +``` + +After this command two new files will be created: `pwd`/profile.dat.truncated and `pwd`/profile2.dat.truncated. + +`--count ` option can also be used to remove N methods from tail of profile. + ### To run some tests: ```sh python3 mcj-edit.py self-test --rw-sha256 -i `pwd`/profile.dat python3 mcj-edit.py self-test --rw -i `pwd`/profile.dat python3 mcj-edit.py self-test --sm -i `pwd`/profile.dat --system-modules-list `pwd`/system_modules.txt +python3 mcj-edit.py self-test --tm -i `pwd`/profile.dat ``` ## pylint diff --git a/src/coreclr/tools/mcj-edit/mcj-edit.py b/src/coreclr/tools/mcj-edit/mcj-edit.py index e0b3ade..751af50 100644 --- a/src/coreclr/tools/mcj-edit/mcj-edit.py +++ b/src/coreclr/tools/mcj-edit/mcj-edit.py @@ -3493,6 +3493,25 @@ class MCJProfile: # pylint: disable=too-many-public-methods if (i >= len(self._modules)): raise InternalError() + moduleFinalLoadLevel = [0] * len(self._modules) + moduleMethodCount = [0] * len(self._modules) + + for info in self._moduleOrMethodInfo: + moduleIndex = info.getModuleIndex() + + if (info.isMethodInfo()): + indexes = info.getAllRefModuleIndexes() + for i in indexes: + moduleMethodCount[i] += 1 + else: + moduleFinalLoadLevel[moduleIndex] = info.getModuleLoadLevel() + + for index, module in enumerate(self._modules): + if (moduleFinalLoadLevel[index] != module.getModuleRecord().getLoadLevel()): + raise InternalError() + if (moduleMethodCount[index] != module.getModuleRecord().getJitMethodCount()): + raise InternalError() + # Print raw header def printRawHeader(self, offsetStr=""): if (offsetStr is None or not issubclass(type(offsetStr), str)): @@ -3719,6 +3738,88 @@ class MCJProfile: # pylint: disable=too-many-public-methods return bytesArr + # Remove requested number of methods from tail of profile + def truncateTail(self, numMethodsToRemove): + if (numMethodsToRemove is None or not issubclass(type(numMethodsToRemove), int)): + raise InternalError() + + methodCount = self._header.getMethodCount() + + if (numMethodsToRemove >= methodCount or numMethodsToRemove <= 0): + raise InternalError() + + removedCountMethods = 0 + removedCountModuleDeps = 0 + + # I. ==== Remove all infos that are met, both method and module ==== + + # Module infos are used by further methods either explicitly or implicitly, + # so they are not needed if there's nothing after them + + for index in range(len(self._moduleOrMethodInfo) - 1, -1, -1): + info = self._moduleOrMethodInfo[index] + + if (info.isMethodInfo()): + if (removedCountMethods == numMethodsToRemove): + break + + removedCountMethods += 1 + + indexes = info.getAllRefModuleIndexes() + for i in indexes: + moduleRecord = self._modules[i].getModuleRecord() + count = moduleRecord.getJitMethodCount() + moduleRecord.setJitMethodCount(count - 1) + else: + removedCountModuleDeps += 1 + + self._moduleOrMethodInfo.pop() + + # II. ==== Remove modules and update final load level ==== + + # Module can be removed if there's no module dependency for such module + + # Fill list of final load level for modules + finalLoadLevelMap = [0] * len(self._modules) + + for info in self._moduleOrMethodInfo: + if (info.isModuleInfo()): + finalLoadLevelMap[info.getModuleIndex()] = info.getModuleLoadLevel() + + # All removed should be at the end of list, find first index + index = 0 + while (index < len(self._modules)): + if (finalLoadLevelMap[index] == 0): + break + self._modules[index].getModuleRecord().setLoadLevel(finalLoadLevelMap[index]) + index += 1 + + indexModuleFirstToRemove = index + + # There should be no mixing of used and not used modules, first part of list should be non 0, last part - 0 + # (there's no way that first module dependency for module with higher index was added before + # first module dependency for module with lower index) + while (index < len(self._modules)): + if (finalLoadLevelMap[index] != 0): + raise InternalError() + if (self._modules[index].getModuleRecord().getJitMethodCount() != 0): + raise InternalError() + index += 1 + + del self._modules[indexModuleFirstToRemove:] + + # III. ==== Update header ==== + + # update stats in headers + self._header.setMethodCount(methodCount - removedCountMethods) + moduleDepCount = self._header.getModuleDepCount() + self._header.setModuleCount(len(self._modules)) + self._header.setModuleDepCount(moduleDepCount - removedCountModuleDeps) + # new profile was not used yet, so drop usage stats + self._header.dropGlobalUsageStats() + + self.verify() + # Split mcj profile in two: app-dependent (app) and app-independent (system) def split(self, systemModules): # pylint: disable=too-many-locals,too-many-statements,too-many-branches if (systemModules is None or not issubclass(type(systemModules), list)): @@ -3971,7 +4072,7 @@ class MCJProfile: # pylint: disable=too-many-public-methods newinfoToAdd.updateModuleIndex(moduleIndexMap) self._moduleOrMethodInfo.append(newinfoToAdd) - # IV. ==== Set stats ==== + # V. ==== Set stats ==== methodCount = 0 moduleDepCount = 0 @@ -4045,6 +4146,35 @@ class CLI: def __init__(self, args): self._args = args + # Remove requested number of methods from tail of profile + def commandTruncate(self): + for filepath in self._args.input: + outFilepath = filepath + ".truncated" + + mcjProfile = MCJProfile.readFromFile(filepath) + numMethods = mcjProfile.getHeader().getMethodCount() + + percentMethodsToRemove = 0 + numMethodsToRemove = 0 + + if ((self._args.percent is None) == (self._args.count is None)): # pylint: disable=no-else-raise + raise MessageError("either '--percent ' or '--count ' option should be passed") + elif (not self._args.percent is None): + percentMethodsToRemove = self._args.percent + numMethodsToRemove = int(numMethods * percentMethodsToRemove / 100) + elif (not self._args.count is None): + numMethodsToRemove = self._args.count + percentMethodsToRemove = int((numMethodsToRemove / numMethods) * 100) + else: + raise InternalError() + + mcjProfile.truncateTail(numMethodsToRemove) + MCJProfile.writeToFile(outFilepath, mcjProfile) + + print("MCJ profile " + filepath + " was truncated by " + str(percentMethodsToRemove) + + "% (" + str(numMethodsToRemove) + " methods), output: " + outFilepath) + print("") + # Split mcj profiles in two: app-dependent (app) and app-independent (system) def commandSplit(self): systemModulesFile = open(self._args.system_modules_list, "r") @@ -4374,8 +4504,10 @@ class CLI: splitSysFirstApp, splitSysFirstSys = mergedSysFirst.split(systemModules) if (depth > depthCheck - and (splitAppFirstApp != prevSplitAppFirstApp or splitAppFirstSys != prevSplitAppFirstSys - or splitSysFirstApp != prevSplitSysFirstApp or splitSysFirstSys != prevSplitSysFirstSys)): + and (splitAppFirstApp != prevSplitAppFirstApp + or splitAppFirstSys != prevSplitAppFirstSys + or splitSysFirstApp != prevSplitSysFirstApp + or splitSysFirstSys != prevSplitSysFirstSys)): isCorrect = False break @@ -4412,6 +4544,31 @@ class CLI: print("Split-merge self test passed for " + filepath) else: print("Split-merge self test failed for " + filepath) + elif (self._args.tm): + # Truncate mcj profile, merge truncated profile with original one, result should match original one + # Repeat this for different truncation percent + for filepath in self._args.input: + isCorrect = True + + mcjProfile = MCJProfile.readFromFile(filepath) + numMethods = mcjProfile.getHeader().getMethodCount() + + for percent in range(30,91,30): + mcjProfile2 = mcjProfile.copy() + + numMethodsToRemove = int(numMethods * percent / 100) + + mcjProfile2.truncateTail(numMethodsToRemove) + mcjProfile2.merge(mcjProfile) + + if (mcjProfile2 != mcjProfile): + isCorrect = False + break + + if (isCorrect): + print("Truncate-merge self test passed for " + filepath) + else: + print("Truncate-merge self test failed for " + filepath) elif (self._args.unit): # TODO: add unit tests pass @@ -4423,13 +4580,17 @@ class CLI: def main(): parser = argparse.ArgumentParser() - commands = "split, merge, verify, find, compare, clean-stats, print, help, self-test" + commands = "truncate, split, merge, verify, find, compare, clean-stats, print, help, self-test" parser.add_argument("command", help="Command to execute: " + commands) # Overall options parser.add_argument("-i", "--input", help="Input mcj profiles", action="append") + # Truncate options + parser.add_argument("--percent", help="Percent of methods to remove from tail of profile", type=int) + parser.add_argument("--count", help="Number of methods to remove from tail of profile", type=int) + # Split options parser.add_argument("--system-modules-list", help="[split], file with app-independent (i.e. system) module names") @@ -4463,13 +4624,16 @@ def main(): parser.add_argument("--rw-sha256", help="[self-test], perform read-write self-test using sha256sum", action="store_true") parser.add_argument("--sm", help="[self-test], perform split-merge self-test", action="store_true") + parser.add_argument("--tm", help="[self-test], perform truncate-merge self-test", action="store_true") parser.add_argument("--unit", help="TODO, [self-test], perform unit testing self-test", action="store_true") args = parser.parse_args() cli = CLI(args) - if (args.command == "split"): + if (args.command == "truncate"): + cli.commandTruncate() + elif (args.command == "split"): cli.commandSplit() elif (args.command == "merge"): cli.commandMerge()