From 41c27ffb5a4145633b321de3441372b4e36f81f7 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 6 Nov 2020 10:06:26 -0800 Subject: [PATCH] Add threshold on number of files / partition in SPMI collection (#44180) * Add check for files count * Fix the OS check * decrese file limit to 1500: * misc fix * Do not upload to azure if mch files are zero size --- eng/pipelines/coreclr/templates/run-superpmi-job.yml | 2 +- src/coreclr/scripts/superpmi-setup.py | 6 +++--- src/coreclr/scripts/superpmi.proj | 8 ++++---- src/coreclr/scripts/superpmi.py | 15 ++++++++------- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-job.yml index 7c7b96e..02c37e8 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-job.yml @@ -105,7 +105,7 @@ jobs: - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml parameters: HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'test/superpmi/$(Kind)/$(_Framework)/$(Architecture)' + HelixType: 'test/superpmi/$(CollectionName)/$(_Framework)/$(Architecture)' HelixAccessToken: $(HelixApiAccessToken) HelixTargetQueues: $(Queue) HelixPreCommands: $(HelixPreCommand) diff --git a/src/coreclr/scripts/superpmi-setup.py b/src/coreclr/scripts/superpmi-setup.py index d119fba..1036580 100644 --- a/src/coreclr/scripts/superpmi-setup.py +++ b/src/coreclr/scripts/superpmi-setup.py @@ -89,7 +89,7 @@ native_binaries_to_ignore = [ "superpmi-shim-counter.dll", "superpmi-shim-simple.dll", ] - +MAX_FILES_COUNT = 1500 def setup_args(args): """ Setup the args for SuperPMI to use. @@ -206,7 +206,7 @@ def first_fit(sorted_by_size, max_size): if file_size < max_size: for p_index in partitions: total_in_curr_par = sum(n for _, n in partitions[p_index]) - if (total_in_curr_par + file_size) < max_size: + if (((total_in_curr_par + file_size) < max_size) and (len(partitions[p_index]) < MAX_FILES_COUNT)): partitions[p_index].append(curr_file) found_bucket = True break @@ -217,7 +217,7 @@ def first_fit(sorted_by_size, max_size): total_size = 0 for p_index in partitions: partition_size = sum(n for _, n in partitions[p_index]) - print("Partition {0}: {1} bytes.".format(p_index, partition_size)) + print("Partition {0}: {1} files with {2} bytes.".format(p_index, len(partitions[p_index]), partition_size)) total_size += partition_size print("Total {0} partitions with {1} bytes.".format(str(len(partitions)), total_size)) diff --git a/src/coreclr/scripts/superpmi.proj b/src/coreclr/scripts/superpmi.proj index c7facc7..7524245 100644 --- a/src/coreclr/scripts/superpmi.proj +++ b/src/coreclr/scripts/superpmi.proj @@ -1,9 +1,9 @@ - + \ - + / @@ -16,7 +16,7 @@ PmiAssembliesPayload - Path that will be sent to helix machine to run collection on PmiAssembliesDirectory - Path on helix machine itself where superpmi.py will discover the sent assemblies. --> - + %HELIX_PYTHONPATH% $(WorkItemDirectory)\pmiAssembliesDirectory %HELIX_WORKITEM_PAYLOAD%\binaries @@ -26,7 +26,7 @@ $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults $(SuperPMIDirectory)\superpmi.py collect --pmi -pmi_location $(SuperPMIDirectory)\pmi.dll - + $HELIX_PYTHONPATH $(WorkItemDirectory)/pmiAssembliesDirectory $HELIX_WORKITEM_PAYLOAD/binaries diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index 150880a..2540559 100755 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -2175,19 +2175,20 @@ def upload_mch(coreclr_args): files = [] for item in coreclr_args.mch_files: - files += get_files_from_path(item, match_func=lambda path: any(path.endswith(extension) for extension in [".mch", ".mct"])) + files += get_files_from_path(item, match_func=lambda path: any(path.endswith(extension) for extension in [".mch"])) + files_to_upload = [] # Special case: walk the files list and for every ".mch" file, check to see that either the associated ".mct" file is already # in the list, or add it if the ".mct" file exists. for file in files.copy(): - if file.endswith(".mch"): + if file.endswith(".mch") and os.stat(file).st_size > 0: + files_to_upload.append(file) mct_file = file + ".mct" - if mct_file not in files: - if os.path.isfile(mct_file): - files.append(mct_file) + if os.path.isfile(mct_file) and os.stat(mct_file).st_size > 0: + files_to_upload.append(mct_file) logging.info("Uploading:") - for item in files: + for item in files_to_upload: logging.info(" %s", item) try: @@ -2205,7 +2206,7 @@ def upload_mch(coreclr_args): total_bytes_uploaded = 0 with TempDir() as temp_location: - for file in files: + for file in files_to_upload: # Zip compress the file we will upload zip_name = os.path.basename(file) + ".zip" zip_path = os.path.join(temp_location, zip_name) -- 2.7.4