2 # Copyright 2010 Google Inc. All Rights Reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 """Provides cross-platform utility fuctions.
19 import platformsettings
20 ip = platformsettings.get_server_ip_address()
22 Functions with "_temporary_" in their name automatically clean-up upon
23 termination (via the atexit module).
25 For the full list of functions, see the bottom of the file.
41 class PlatformSettingsError(Exception):
42 """Module catch-all error."""
46 class DnsReadError(PlatformSettingsError):
47 """Raised when unable to read DNS settings."""
51 class DnsUpdateError(PlatformSettingsError):
52 """Raised when unable to update DNS settings."""
56 class NotAdministratorError(PlatformSettingsError):
57 """Raised when not running as administrator."""
61 class CalledProcessError(PlatformSettingsError):
62 """Raised when a _check_output() process returns a non-zero exit status."""
63 def __init__(self, returncode, cmd):
64 self.returncode = returncode
68 return 'Command "%s" returned non-zero exit status %d' % (
69 ' '.join(self.cmd), self.returncode)
72 def _check_output(*args):
73 """Run Popen(*args) and return its output as a byte string.
75 Python 2.7 has subprocess.check_output. This is essentially the same
76 except that, as a convenience, all the positional args are used as
80 *args: sequence of program arguments
82 CalledProcessError if the program returns non-zero exit status.
84 output as a byte string.
86 command_args = [str(a) for a in args]
87 logging.debug(' '.join(command_args))
88 process = subprocess.Popen(command_args,
89 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
90 output = process.communicate()[0]
91 retcode = process.poll()
93 raise CalledProcessError(retcode, command_args)
97 class _BasePlatformSettings(object):
99 def get_system_logging_handler(self):
100 """Return a handler for the logging module (optional)."""
103 def rerun_as_administrator(self):
104 """If needed, rerun the program with administrative privileges.
106 Raises NotAdministratorError if unable to rerun.
111 """Return the current time in seconds as a floating point number."""
114 def get_server_ip_address(self, is_server_mode=False):
115 """Returns the IP address to use for dnsproxy and ipfw."""
117 return socket.gethostbyname(socket.gethostname())
120 def get_httpproxy_ip_address(self, is_server_mode=False):
121 """Returns the IP address to use for httpproxy."""
127 raise NotImplementedError
129 def ipfw(self, *args):
130 ipfw_cmd = self._ipfw_cmd() + args
131 return _check_output(*ipfw_cmd)
133 def ping_rtt(self, hostname):
134 """Pings the hostname by calling the OS system ping command.
135 Also stores the result internally.
138 hostname: hostname of the server to be pinged
140 round trip time to the server in seconds, or 0 if unable to calculate RTT
142 raise NotImplementedError
147 def _set_cwnd(self, args):
150 def set_temporary_tcp_init_cwnd(self, cwnd):
152 original_cwnd = self._get_cwnd()
153 if original_cwnd is None:
154 raise PlatformSettingsError('Unable to get current tcp init_cwnd.')
155 if cwnd == original_cwnd:
156 logging.info('TCP init_cwnd already set to target value: %s', cwnd)
159 if self._get_cwnd() == cwnd:
160 logging.info('Changed cwnd to %s', cwnd)
161 atexit.register(self._set_cwnd, original_cwnd)
163 logging.error('Unable to update cwnd to %s', cwnd)
165 def setup_temporary_loopback_config(self):
166 """Setup the loopback interface similar to real interface.
168 We use loopback for much of our testing, and on some systems, loopback
169 behaves differently from real interfaces.
171 logging.error('Platform does not support loopback configuration.')
173 def _get_primary_nameserver(self):
174 raise NotImplementedError
176 def _set_primary_nameserver(self):
177 raise NotImplementedError
179 def get_original_primary_nameserver(self):
180 if not hasattr(self, '_original_nameserver'):
181 self._original_nameserver = self._get_primary_nameserver()
182 logging.info('Saved original primary DNS nameserver: %s',
183 self._original_nameserver)
184 return self._original_nameserver
186 def set_temporary_primary_nameserver(self, nameserver):
187 orig_nameserver = self.get_original_primary_nameserver()
188 self._set_primary_nameserver(nameserver)
189 if self._get_primary_nameserver() == nameserver:
190 logging.info('Changed temporary primary nameserver to %s', nameserver)
191 atexit.register(self._set_primary_nameserver, orig_nameserver)
193 raise self._get_dns_update_error()
196 class _PosixPlatformSettings(_BasePlatformSettings):
197 PING_PATTERN = r'rtt min/avg/max/mdev = \d+\.\d+/(\d+\.\d+)/\d+\.\d+/\d+\.\d+'
198 PING_CMD = ('ping', '-c', '3', '-i', '0.2', '-W', '1')
199 # For OsX Lion non-root:
200 PING_RESTRICTED_CMD = ('ping', '-c', '1', '-i', '1', '-W', '1')
201 SUDO_PATH = '/usr/bin/sudo'
203 def rerun_as_administrator(self):
204 """If needed, rerun the program with administrative privileges.
206 Raises NotAdministratorError if unable to rerun.
208 if os.geteuid() != 0:
209 logging.warn('Rerunning with sudo: %s', sys.argv)
210 os.execv(self.SUDO_PATH, ['--'] + sys.argv)
213 for ipfw_path in ['/usr/local/sbin/ipfw', '/sbin/ipfw']:
214 if os.path.exists(ipfw_path):
215 ipfw_cmd = (self.SUDO_PATH, ipfw_path)
216 self._ipfw_cmd = lambda: ipfw_cmd # skip rechecking paths
218 raise PlatformSettingsError('ipfw not found.')
220 def _ping(self, hostname):
221 """Return ping output or None if ping fails.
223 Initially pings 'localhost' to test for ping command that works.
224 If the tests fails, subsequent calls will return None without calling ping.
227 hostname: host to ping
229 ping stdout string, or None if ping unavailable
231 CalledProcessError if ping returns non-zero exit
233 if not hasattr(self, 'ping_cmd'):
234 test_host = 'localhost'
235 for self.ping_cmd in (self.PING_CMD, self.PING_RESTRICTED_CMD):
237 if self._ping(test_host):
239 except (CalledProcessError, OSError) as e:
242 logging.critical('Ping configuration failed: %s', last_ping_error)
245 cmd = list(self.ping_cmd) + [hostname]
246 return self._check_output(*cmd)
249 def ping_rtt(self, hostname):
250 """Pings the hostname by calling the OS system ping command.
253 hostname: hostname of the server to be pinged
255 round trip time to the server in milliseconds, or 0 if unavailable
260 output = self._ping(hostname)
261 except CalledProcessError as e:
262 logging.critical('Ping failed: %s', e)
264 match = re.search(self.PING_PATTERN, output)
266 rtt = float(match.groups()[0])
268 logging.warning('Unable to ping %s: %s', hostname, output)
272 def _get_dns_update_error(self):
273 return DnsUpdateError('Did you run under sudo?')
276 def _sysctl(cls, *args, **kwargs):
278 if kwargs.get('use_sudo'):
279 sysctl_args.append(cls.SUDO_PATH)
280 sysctl_args.append('/usr/sbin/sysctl')
281 if not os.path.exists(sysctl_args[-1]):
282 sysctl_args[-1] = '/sbin/sysctl'
283 sysctl_args.extend(str(a) for a in args)
284 sysctl = subprocess.Popen(
285 sysctl_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
286 stdout = sysctl.communicate()[0]
287 return sysctl.returncode, stdout
289 def has_sysctl(self, name):
290 if not hasattr(self, 'has_sysctl_cache'):
291 self.has_sysctl_cache = {}
292 if name not in self.has_sysctl_cache:
293 self.has_sysctl_cache[name] = self._sysctl(name)[0] == 0
294 return self.has_sysctl_cache[name]
296 def set_sysctl(self, name, value):
297 rv = self._sysctl('%s=%s' % (name, value), use_sudo=True)[0]
299 logging.error('Unable to set sysctl %s: %s', name, rv)
301 def get_sysctl(self, name):
302 rv, value = self._sysctl('-n', name)
306 logging.error('Unable to get sysctl %s: %s', name, rv)
309 def _check_output(self, *args):
310 """Allow tests to override this."""
311 return _check_output(*args)
314 class _OsxPlatformSettings(_PosixPlatformSettings):
315 LOCAL_SLOWSTART_MIB_NAME = 'net.inet.tcp.local_slowstart_flightsize'
317 def _scutil(self, cmd):
318 scutil = subprocess.Popen(
319 ['/usr/sbin/scutil'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
320 return scutil.communicate(cmd)[0]
322 def _ifconfig(self, *args):
323 return _check_output(self.SUDO_PATH, '/sbin/ifconfig', *args)
325 def set_sysctl(self, name, value):
326 rv = self._sysctl('-w', '%s=%s' % (name, value), use_sudo=True)[0]
328 logging.error('Unable to set sysctl %s: %s', name, rv)
331 return int(self.get_sysctl(self.LOCAL_SLOWSTART_MIB_NAME))
333 def _set_cwnd(self, size):
334 self.set_sysctl(self.LOCAL_SLOWSTART_MIB_NAME, size)
336 def _get_loopback_mtu(self):
337 config = self._ifconfig('lo0')
338 match = re.search(r'\smtu\s+(\d+)', config)
339 return int(match.group(1)) if match else None
341 def setup_temporary_loopback_config(self):
342 """Configure loopback to temporarily use reasonably sized frames.
344 OS X uses jumbo frames by default (16KB).
346 TARGET_LOOPBACK_MTU = 1500
347 original_mtu = self._get_loopback_mtu()
348 if original_mtu is None:
349 logging.error('Unable to read loopback mtu. Setting left unchanged.')
351 if original_mtu == TARGET_LOOPBACK_MTU:
352 logging.debug('Loopback MTU already has target value: %d', original_mtu)
354 self._ifconfig('lo0', 'mtu', TARGET_LOOPBACK_MTU)
355 if self._get_loopback_mtu() == TARGET_LOOPBACK_MTU:
356 logging.debug('Set loopback MTU to %d (was %d)',
357 TARGET_LOOPBACK_MTU, original_mtu)
358 atexit.register(self._ifconfig, 'lo0', 'mtu', original_mtu)
360 logging.error('Unable to change loopback MTU from %d to %d',
361 original_mtu, TARGET_LOOPBACK_MTU)
363 def _get_dns_service_key(self):
364 output = self._scutil('show State:/Network/Global/IPv4')
365 lines = output.split('\n')
367 key_value = line.split(' : ')
368 if key_value[0] == ' PrimaryService':
369 return 'State:/Network/Service/%s/DNS' % key_value[1]
370 raise DnsReadError('Unable to find DNS service key: %s', output)
372 def _get_primary_nameserver(self):
373 output = self._scutil('show %s' % self._get_dns_service_key())
375 br'ServerAddresses\s+:\s+<array>\s+{\s+0\s+:\s+((\d{1,3}\.){3}\d{1,3})',
378 return match.group(1)
380 raise DnsReadError('Unable to find primary DNS server: %s', output)
382 def _set_primary_nameserver(self, dns):
383 command = '\n'.join([
385 'd.add ServerAddresses * %s' % dns,
386 'set %s' % self._get_dns_service_key()
388 self._scutil(command)
391 class _LinuxPlatformSettings(_PosixPlatformSettings):
392 """The following thread recommends a way to update DNS on Linux:
394 http://ubuntuforums.org/showthread.php?t=337553
396 sudo cp /etc/dhcp3/dhclient.conf /etc/dhcp3/dhclient.conf.bak
397 sudo gedit /etc/dhcp3/dhclient.conf
398 #prepend domain-name-servers 127.0.0.1;
399 prepend domain-name-servers 208.67.222.222, 208.67.220.220;
401 prepend domain-name-servers 208.67.222.222, 208.67.220.220;
402 request subnet-mask, broadcast-address, time-offset, routers,
403 domain-name, domain-name-servers, host-name,
404 netbios-name-servers, netbios-scope;
405 #require subnet-mask, domain-name-servers;
407 sudo /etc/init.d/networking restart
409 The code below does not try to change dchp and does not restart networking.
410 Update this as needed to make it more robust on more systems.
412 RESOLV_CONF = '/etc/resolv.conf'
413 ROUTE_RE = re.compile('initcwnd (\d+)')
414 TCP_BASE_MSS = 'net.ipv4.tcp_base_mss'
415 TCP_MTU_PROBING = 'net.ipv4.tcp_mtu_probing'
417 def _get_default_route_line(self):
418 stdout = self._check_output('ip', 'route')
419 for line in stdout.split('\n'):
420 if line.startswith('default'):
424 def _set_cwnd(self, cwnd):
425 default_line = self._get_default_route_line()
427 'ip', 'route', 'change', default_line, 'initcwnd', str(cwnd))
430 default_line = self._get_default_route_line()
431 m = self.ROUTE_RE.search(default_line)
433 return int(m.group(1))
434 # If 'initcwnd' wasn't found, then 0 means it's the system default.
437 def setup_temporary_loopback_config(self):
438 """Setup Linux to temporarily use reasonably sized frames.
440 Linux uses jumbo frames by default (16KB), using the combination
441 of MTU probing and a base MSS makes it use normal sized packets.
443 The reason this works is because tcp_base_mss is only used when MTU
444 probing is enabled. And since we're using the max value, it will
445 always use the reasonable size. This is relevant for server-side realism.
446 The client-side will vary depending on the client TCP stack config.
448 ENABLE_MTU_PROBING = 2
449 original_probing = self.get_sysctl(self.TCP_MTU_PROBING)
450 self.set_sysctl(self.TCP_MTU_PROBING, ENABLE_MTU_PROBING)
451 atexit.register(self.set_sysctl, self.TCP_MTU_PROBING, original_probing)
454 original_mss = self.get_sysctl(self.TCP_BASE_MSS)
455 self.set_sysctl(self.TCP_BASE_MSS, TCP_FULL_MSS)
456 atexit.register(self.set_sysctl, self.TCP_BASE_MSS, original_mss)
458 def _write_resolve_conf(self, dns):
459 is_first_nameserver_replaced = False
460 # The fileinput module uses sys.stdout as the edited file output.
461 for line in fileinput.input(self.RESOLV_CONF, inplace=1, backup='.bak'):
462 if line.startswith('nameserver ') and not is_first_nameserver_replaced:
463 print 'nameserver %s' % dns
464 is_first_nameserver_replaced = True
467 if not is_first_nameserver_replaced:
468 raise DnsUpdateError('Could not find a suitable nameserver entry in %s' %
471 def _get_primary_nameserver(self):
473 resolv_file = open(self.RESOLV_CONF)
476 for line in resolv_file:
477 if line.startswith('nameserver '):
478 return line.split()[1]
481 def _set_primary_nameserver(self, dns):
482 """Replace the first nameserver entry with the one given."""
484 self._write_resolve_conf(dns)
486 if 'Permission denied' in e:
487 raise self._get_dns_update_error()
491 class _WindowsPlatformSettings(_BasePlatformSettings):
493 def get_system_logging_handler(self):
494 """Return a handler for the logging module (optional).
496 For Windows, output can be viewed with DebugView.
497 http://technet.microsoft.com/en-us/sysinternals/bb896647.aspx
500 output_debug_string = ctypes.windll.kernel32.OutputDebugStringA
501 output_debug_string.argtypes = [ctypes.c_char_p]
502 class DebugViewHandler(logging.Handler):
503 def emit(self, record):
504 output_debug_string('[wpr] ' + self.format(record))
505 return DebugViewHandler()
507 def rerun_as_administrator(self):
508 """If needed, rerun the program with administrative privileges.
510 Raises NotAdministratorError if unable to rerun.
513 if not ctypes.windll.shell32.IsUserAnAdmin():
514 raise NotAdministratorError('Rerun with administrator privileges.')
515 #os.execv('runas', sys.argv) # TODO: replace needed Windows magic
518 """Return the current time in seconds as a floating point number.
520 From time module documentation:
521 On Windows, this function [time.clock()] returns wall-clock
522 seconds elapsed since the first call to this function, as a
523 floating point number, based on the Win32 function
524 QueryPerformanceCounter(). The resolution is typically better
525 than one microsecond.
529 def _arp(self, *args):
530 return _check_output('arp', *args)
532 def _route(self, *args):
533 return _check_output('route', *args)
535 def _ipconfig(self, *args):
536 return _check_output('ipconfig', *args)
538 def _get_mac_address(self, ip):
539 """Return the MAC address for the given ip."""
540 ip_re = re.compile(r'^\s*IP(?:v4)? Address[ .]+:\s+([0-9.]+)')
541 for line in self._ipconfig('/all').splitlines():
542 if line[:1].isalnum():
547 ip_match = ip_re.match(line)
549 current_ip = ip_match.group(1)
550 elif line.startswith('Physical Address'):
551 current_mac = line.split(':', 1)[1].lstrip()
552 if current_ip == ip and current_mac:
556 def setup_temporary_loopback_config(self):
557 """On Windows, temporarily route the server ip to itself."""
558 ip = self.get_server_ip_address()
559 mac_address = self._get_mac_address(ip)
561 self._arp('-s', ip, self.mac_address)
562 self._route('add', ip, ip, 'mask', '255.255.255.255')
563 atexit.register(self._arp, '-d', ip)
564 atexit.register(self._route, 'delete', ip, ip, 'mask', '255.255.255.255')
566 logging.warn('Unable to configure loopback: MAC address not found.')
567 # TODO(slamm): Configure cwnd, MTU size
569 def _get_dns_update_error(self):
570 return DnsUpdateError('Did you run as administrator?')
572 def _netsh_show_dns(self):
573 """Return DNS information:
576 Configuration for interface "Local Area Connection 3"
577 DNS servers configured through DHCP: None
578 Register with which suffix: Primary only
580 Configuration for interface "Wireless Network Connection 2"
581 DNS servers configured through DHCP: 192.168.1.1
582 Register with which suffix: Primary only
584 return _check_output('netsh', 'interface', 'ip', 'show', 'dns')
586 def _get_primary_nameserver(self):
587 match = re.search(r':\s+(\d+\.\d+\.\d+\.\d+)', self._netsh_show_dns())
588 return match.group(1) if match else None
590 def _set_primary_nameserver(self, dns):
592 Set objWMIService = GetObject( _
593 "winmgmts:{impersonationLevel=impersonate}!\\\\.\\root\\cimv2")
594 Set colNetCards = objWMIService.ExecQuery( _
595 "Select * From Win32_NetworkAdapterConfiguration Where IPEnabled = True")
596 For Each objNetCard in colNetCards
597 arrDNSServers = Array("%s")
598 objNetCard.SetDNSServerSearchOrder(arrDNSServers)
601 vbs_file = tempfile.NamedTemporaryFile(suffix='.vbs', delete=False)
604 subprocess.check_call(['cscript', '//nologo', vbs_file.name])
605 os.remove(vbs_file.name)
608 class _WindowsXpPlatformSettings(_WindowsPlatformSettings):
610 return (r'third_party\ipfw_win32\ipfw.exe',)
613 def _new_platform_settings(system, release):
614 """Make a new instance of PlatformSettings for the current system."""
615 if system == 'Darwin':
616 return _OsxPlatformSettings()
617 if system == 'Linux':
618 return _LinuxPlatformSettings()
619 if system == 'Windows' and release == 'XP':
620 return _WindowsXpPlatformSettings()
621 if system == 'Windows':
622 return _WindowsPlatformSettings()
623 raise NotImplementedError('Sorry %s %s is not supported.' % (system, release))
626 # Create one instance of the platform-specific settings and
627 # make the functions available at the module-level.
628 _inst = _new_platform_settings(platform.system(), platform.release())
630 get_system_logging_handler = _inst.get_system_logging_handler
631 rerun_as_administrator = _inst.rerun_as_administrator
634 get_server_ip_address = _inst.get_server_ip_address
635 get_httpproxy_ip_address = _inst.get_httpproxy_ip_address
637 ping_rtt = _inst.ping_rtt
638 set_temporary_tcp_init_cwnd = _inst.set_temporary_tcp_init_cwnd
639 setup_temporary_loopback_config = _inst.setup_temporary_loopback_config
641 get_original_primary_nameserver = _inst.get_original_primary_nameserver
642 set_temporary_primary_nameserver = _inst.set_temporary_primary_nameserver