build/toolchain/get_concurrent_links.py

   1 #!/usr/bin/env python3
   2 # Copyright 2014 The Chromium Authors
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 # This script computs the number of concurrent links we want to run in the build
   7 # as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP.
   8
   9 import argparse
  10 import multiprocessing
  11 import os
  12 import re
  13 import subprocess
  14 import sys
  15
  16 sys.path.insert(1, os.path.join(os.path.dirname(__file__), '..'))
  17 import gn_helpers
  18
  19
  20 def _GetTotalMemoryInBytes():
  21   if sys.platform in ('win32', 'cygwin'):
  22     import ctypes
  23
  24     class MEMORYSTATUSEX(ctypes.Structure):
  25       _fields_ = [
  26           ("dwLength", ctypes.c_ulong),
  27           ("dwMemoryLoad", ctypes.c_ulong),
  28           ("ullTotalPhys", ctypes.c_ulonglong),
  29           ("ullAvailPhys", ctypes.c_ulonglong),
  30           ("ullTotalPageFile", ctypes.c_ulonglong),
  31           ("ullAvailPageFile", ctypes.c_ulonglong),
  32           ("ullTotalVirtual", ctypes.c_ulonglong),
  33           ("ullAvailVirtual", ctypes.c_ulonglong),
  34           ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
  35       ]
  36
  37     stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
  38     ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
  39     return stat.ullTotalPhys
  40   elif sys.platform.startswith('linux'):
  41     if os.path.exists("/proc/meminfo"):
  42       with open("/proc/meminfo") as meminfo:
  43         memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
  44         for line in meminfo:
  45           match = memtotal_re.match(line)
  46           if not match:
  47             continue
  48           return float(match.group(1)) * 2**10
  49   elif sys.platform == 'darwin':
  50     try:
  51       return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
  52     except Exception:
  53       return 0
  54   # TODO(scottmg): Implement this for other platforms.
  55   return 0
  56
  57
  58 def _GetDefaultConcurrentLinks(per_link_gb, reserve_gb, thin_lto_type,
  59                                secondary_per_link_gb, override_ram_in_gb):
  60   explanation = []
  61   explanation.append(
  62       'per_link_gb={} reserve_gb={} secondary_per_link_gb={}'.format(
  63           per_link_gb, reserve_gb, secondary_per_link_gb))
  64   if override_ram_in_gb:
  65     mem_total_gb = override_ram_in_gb
  66   else:
  67     mem_total_gb = float(_GetTotalMemoryInBytes()) / 2**30
  68   adjusted_mem_total_gb = max(0, mem_total_gb - reserve_gb)
  69
  70   # Ensure that there is at least as many links allocated for the secondary as
  71   # there is for the primary. The secondary link usually uses fewer gbs.
  72   mem_cap = int(
  73       max(1, adjusted_mem_total_gb / (per_link_gb + secondary_per_link_gb)))
  74
  75   try:
  76     cpu_count = multiprocessing.cpu_count()
  77   except:
  78     cpu_count = 1
  79
  80   # A local LTO links saturate all cores, but only for some amount of the link.
  81   # Goma LTO runs LTO codegen on goma, only run one of these tasks at once.
  82   cpu_cap = cpu_count
  83   if thin_lto_type is not None:
  84     if thin_lto_type == 'goma':
  85       cpu_cap = 1
  86     else:
  87       assert thin_lto_type == 'local'
  88       cpu_cap = min(cpu_count, 6)
  89
  90   explanation.append(
  91       'cpu_count={} cpu_cap={} mem_total_gb={:.1f}GiB adjusted_mem_total_gb={:.1f}GiB'
  92       .format(cpu_count, cpu_cap, mem_total_gb, adjusted_mem_total_gb))
  93
  94   num_links = min(mem_cap, cpu_cap)
  95   if num_links == cpu_cap:
  96     if cpu_cap == cpu_count:
  97       reason = 'cpu_count'
  98     else:
  99       reason = 'cpu_cap (thinlto)'
 100   else:
 101     reason = 'RAM'
 102
 103   # static link see too many open files if we have many concurrent links.
 104   # ref: http://b/233068481
 105   if num_links > 30:
 106     num_links = 30
 107     reason = 'nofile'
 108
 109   explanation.append('concurrent_links={}  (reason: {})'.format(
 110       num_links, reason))
 111
 112   # Use remaining RAM for a secondary pool if needed.
 113   if secondary_per_link_gb:
 114     mem_remaining = adjusted_mem_total_gb - num_links * per_link_gb
 115     secondary_size = int(max(0, mem_remaining / secondary_per_link_gb))
 116     if secondary_size > cpu_count:
 117       secondary_size = cpu_count
 118       reason = 'cpu_count'
 119     else:
 120       reason = 'mem_remaining={:.1f}GiB'.format(mem_remaining)
 121     explanation.append('secondary_size={} (reason: {})'.format(
 122         secondary_size, reason))
 123   else:
 124     secondary_size = 0
 125
 126   return num_links, secondary_size, explanation
 127
 128
 129 def main():
 130   parser = argparse.ArgumentParser()
 131   parser.add_argument('--mem_per_link_gb', type=int, default=8)
 132   parser.add_argument('--reserve_mem_gb', type=int, default=0)
 133   parser.add_argument('--secondary_mem_per_link', type=int, default=0)
 134   parser.add_argument('--override-ram-in-gb-for-testing', type=float, default=0)
 135   parser.add_argument('--thin-lto')
 136   options = parser.parse_args()
 137
 138   primary_pool_size, secondary_pool_size, explanation = (
 139       _GetDefaultConcurrentLinks(options.mem_per_link_gb,
 140                                  options.reserve_mem_gb, options.thin_lto,
 141                                  options.secondary_mem_per_link,
 142                                  options.override_ram_in_gb_for_testing))
 143   if options.override_ram_in_gb_for_testing:
 144     print('primary={} secondary={} explanation={}'.format(
 145         primary_pool_size, secondary_pool_size, explanation))
 146   else:
 147     sys.stdout.write(
 148         gn_helpers.ToGNString({
 149             'primary_pool_size': primary_pool_size,
 150             'secondary_pool_size': secondary_pool_size,
 151             'explanation': explanation,
 152         }))
 153   return 0
 154
 155
 156 if __name__ == '__main__':
 157   sys.exit(main())