From 49b782b2cb09c80a8c476287509a4f566cc597d2 Mon Sep 17 00:00:00 2001 From: Jane Xu Date: Thu, 26 Aug 2021 15:42:00 -0700 Subject: [PATCH] Add shard number to print_test_stats.py upload name (#64055) Summary: Now that the render test results job is gone, each shard on GHA is uploading a JSON test stats report. To ensure differentiation, this PR includes the shard number in the report name. Pull Request resolved: https://github.com/pytorch/pytorch/pull/64055 Reviewed By: iramazanli Differential Revision: D30586869 Pulled By: janeyx99 fbshipit-source-id: fd19f347131deec51486bb0795e4e13ac19bc71a --- tools/stats/print_test_stats.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/stats/print_test_stats.py b/tools/stats/print_test_stats.py index 71df463..1f4c33e 100755 --- a/tools/stats/print_test_stats.py +++ b/tools/stats/print_test_stats.py @@ -781,14 +781,16 @@ def assemble_s3_object( def send_report_to_s3(head_report: Version2Report) -> None: job = os.getenv('JOB_BASE_NAME', os.environ.get('CIRCLE_JOB')) + # SHARD_NUMBER is specific to GHA jobs, as the shard number would be included in CIRCLE_JOB already + shard = os.environ.get('SHARD_NUMBER', '') sha1 = os.environ.get('CIRCLE_SHA1') branch = os.environ.get('CIRCLE_BRANCH', '') now = datetime.datetime.utcnow().isoformat() if branch not in ['master', 'nightly'] and not branch.startswith("release/"): pr = os.environ.get('CIRCLE_PR_NUMBER', 'unknown') - key = f'pr_test_time/{pr}/{sha1}/{job}/{now}Z.json.bz2' # Z meaning UTC + key = f'pr_test_time/{pr}/{sha1}/{job}{shard}/{now}Z.json.bz2' # Z meaning UTC else: - key = f'test_time/{sha1}/{job}/{now}Z.json.bz2' # Z meaning UTC + key = f'test_time/{sha1}/{job}{shard}/{now}Z.json.bz2' # Z meaning UTC obj = get_S3_object_from_bucket('ossci-metrics', key) # use bz2 because the results are smaller than gzip, and the # compression time penalty we pay is only about half a second for -- 2.7.4