3 # Copyright 2013 The Chromium Authors. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file.
7 """A class to keep track of devices across builds and report state."""
24 sys.path.append(os.path.join(os.path.dirname(__file__),
25 os.pardir, os.pardir, 'util', 'lib',
27 import perf_tests_results_helper # pylint: disable=F0401
29 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
30 from pylib import android_commands
31 from pylib import constants
32 from pylib.cmd_helper import GetCmdOutput
33 from pylib.device import device_blacklist
34 from pylib.device import device_errors
35 from pylib.device import device_list
36 from pylib.device import device_utils
38 def DeviceInfo(serial, options):
39 """Gathers info on a device via various adb calls.
42 serial: The serial of the attached device to construct info about.
45 Tuple of device type, build id, report as a string, error messages, and
46 boolean indicating whether or not device can be used for testing.
49 device_adb = device_utils.DeviceUtils(serial)
50 device_type = device_adb.GetProp('ro.build.product')
51 device_build = device_adb.GetProp('ro.build.id')
52 device_build_type = device_adb.GetProp('ro.build.type')
53 device_product_name = device_adb.GetProp('ro.product.name')
56 battery_info = device_adb.old_interface.GetBatteryInfo()
57 except Exception as e:
59 logging.error('Unable to obtain battery info for %s, %s', serial, e)
61 def _GetData(re_expression, line, lambda_function=lambda x:x):
64 found = re.findall(re_expression, line)
65 if found and len(found):
66 return lambda_function(found[0])
69 battery_level = int(battery_info.get('level', 100))
70 imei_slice = _GetData('Device ID = (\d+)',
71 device_adb.old_interface.GetSubscriberInfo(),
73 report = ['Device %s (%s)' % (serial, device_type),
75 (device_build, device_adb.GetProp('ro.build.fingerprint')),
76 ' Current Battery Service state: ',
77 '\n'.join([' %s: %s' % (k, v)
78 for k, v in battery_info.iteritems()]),
79 ' IMEI slice: %s' % imei_slice,
80 ' Wifi IP: %s' % device_adb.GetProp('dhcp.wlan0.ipaddress'),
85 if battery_level < 15:
86 errors += ['Device critically low in battery. Turning off device.']
88 if not options.no_provisioning_check:
89 setup_wizard_disabled = (
90 device_adb.GetProp('ro.setupwizard.mode') == 'DISABLED')
91 if not setup_wizard_disabled and device_build_type != 'user':
92 errors += ['Setup wizard not disabled. Was it provisioned correctly?']
93 if (device_product_name == 'mantaray' and
94 battery_info.get('AC powered', None) != 'true'):
95 errors += ['Mantaray device not connected to AC power.']
97 # Turn off devices with low battery.
98 if battery_level < 15:
100 device_adb.EnableRoot()
101 except device_errors.CommandFailedError as e:
102 # Attempt shutdown anyway.
103 # TODO(jbudorick) Handle this exception appropriately after interface
104 # conversions are finished.
105 logging.error(str(e))
106 device_adb.old_interface.Shutdown()
107 full_report = '\n'.join(report)
108 return device_type, device_build, battery_level, full_report, errors, dev_good
111 def CheckForMissingDevices(options, adb_online_devs):
112 """Uses file of previous online devices to detect broken phones.
115 options: out_dir parameter of options argument is used as the base
116 directory to load and update the cache file.
117 adb_online_devs: A list of serial numbers of the currently visible
118 and online attached devices.
120 # TODO(navabi): remove this once the bug that causes different number
121 # of devices to be detected between calls is fixed.
122 logger = logging.getLogger()
123 logger.setLevel(logging.INFO)
125 out_dir = os.path.abspath(options.out_dir)
127 # last_devices denotes all known devices prior to this run
128 last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME)
129 last_missing_devices_path = os.path.join(out_dir,
130 device_list.LAST_MISSING_DEVICES_FILENAME)
132 last_devices = device_list.GetPersistentDeviceList(last_devices_path)
134 # Ignore error, file might not exist
138 last_missing_devices = device_list.GetPersistentDeviceList(
139 last_missing_devices_path)
141 last_missing_devices = []
143 missing_devs = list(set(last_devices) - set(adb_online_devs))
144 new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
146 if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
147 logging.info('new_missing_devs %s' % new_missing_devs)
148 devices_missing_msg = '%d devices not detected.' % len(missing_devs)
149 bb_annotations.PrintSummaryText(devices_missing_msg)
151 from_address = 'chrome-bot@chromium.org'
152 to_addresses = ['chrome-labs-tech-ticket@google.com',
153 'chrome-android-device-alert@google.com']
154 cc_addresses = ['chrome-android-device-alert@google.com']
155 subject = 'Devices offline on %s, %s, %s' % (
156 os.environ.get('BUILDBOT_SLAVENAME'),
157 os.environ.get('BUILDBOT_BUILDERNAME'),
158 os.environ.get('BUILDBOT_BUILDNUMBER'))
159 msg = ('Please reboot the following devices:\n%s' %
160 '\n'.join(map(str,new_missing_devs)))
161 SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
163 all_known_devices = list(set(adb_online_devs) | set(last_devices))
164 device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
165 device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs)
167 if not all_known_devices:
168 # This can happen if for some reason the .last_devices file is not
169 # present or if it was empty.
170 return ['No online devices. Have any devices been plugged in?']
172 devices_missing_msg = '%d devices not detected.' % len(missing_devs)
173 bb_annotations.PrintSummaryText(devices_missing_msg)
175 # TODO(navabi): Debug by printing both output from GetCmdOutput and
176 # GetAttachedDevices to compare results.
177 crbug_link = ('https://code.google.com/p/chromium/issues/entry?summary='
178 '%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
179 (urllib.quote('Device Offline'),
180 urllib.quote('Buildbot: %s %s\n'
182 '(please don\'t change any labels)' %
183 (os.environ.get('BUILDBOT_BUILDERNAME'),
184 os.environ.get('BUILDBOT_SLAVENAME'),
185 os.environ.get('BUILDBOT_BUILDNUMBER')))))
186 return ['Current online devices: %s' % adb_online_devs,
187 '%s are no longer visible. Were they removed?\n' % missing_devs,
189 '@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link,
190 'Cache file: %s\n\n' % last_devices_path,
191 'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
192 'adb devices(GetAttachedDevices): %s' % adb_online_devs]
194 new_devs = set(adb_online_devs) - set(last_devices)
195 if new_devs and os.path.exists(last_devices_path):
196 bb_annotations.PrintWarning()
197 bb_annotations.PrintSummaryText(
198 '%d new devices detected' % len(new_devs))
199 print ('New devices detected %s. And now back to your '
200 'regularly scheduled program.' % list(new_devs))
203 def SendEmail(from_address, to_addresses, cc_addresses, subject, msg):
204 msg_body = '\r\n'.join(['From: %s' % from_address,
205 'To: %s' % ', '.join(to_addresses),
206 'CC: %s' % ', '.join(cc_addresses),
207 'Subject: %s' % subject, '', msg])
209 server = smtplib.SMTP('localhost')
210 server.sendmail(from_address, to_addresses, msg_body)
212 except Exception as e:
213 print 'Failed to send alert email. Error: %s' % e
217 if not os.path.isfile('/usr/bin/restart_usb'):
218 print ('ERROR: Could not restart usb. /usr/bin/restart_usb not installed '
219 'on host (see BUG=305769).')
222 lsusb_proc = bb_utils.SpawnCmd(['lsusb'], stdout=subprocess.PIPE)
223 lsusb_output, _ = lsusb_proc.communicate()
224 if lsusb_proc.returncode:
225 print ('Error: Could not get list of USB ports (i.e. lsusb).')
226 return lsusb_proc.returncode
228 usb_devices = [re.findall('Bus (\d\d\d) Device (\d\d\d)', lsusb_line)[0]
229 for lsusb_line in lsusb_output.strip().split('\n')]
232 # Walk USB devices from leaves up (i.e reverse sorted) restarting the
233 # connection. If a parent node (e.g. usb hub) is restarted before the
234 # devices connected to it, the (bus, dev) for the hub can change, making the
235 # output we have wrong. This way we restart the devices before the hub.
236 for (bus, dev) in reversed(sorted(usb_devices)):
237 # Can not restart root usb connections
239 return_code = bb_utils.RunCmd(['/usr/bin/restart_usb', bus, dev])
241 print 'Error restarting USB device /dev/bus/usb/%s/%s' % (bus, dev)
242 all_restarted = False
244 print 'Restarted USB device /dev/bus/usb/%s/%s' % (bus, dev)
251 for p in psutil.process_iter():
255 except (psutil.error.NoSuchProcess, psutil.error.AccessDenied):
258 for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
259 for p in GetAllAdb():
261 print 'kill %d %d (%s [%s])' % (sig, p.pid, p.name,
264 except (psutil.error.NoSuchProcess, psutil.error.AccessDenied):
266 for p in GetAllAdb():
268 print 'Unable to kill %d (%s [%s])' % (p.pid, p.name, ' '.join(p.cmdline))
269 except (psutil.error.NoSuchProcess, psutil.error.AccessDenied):
274 parser = optparse.OptionParser()
275 parser.add_option('', '--out-dir',
276 help='Directory where the device path is stored',
277 default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
278 parser.add_option('--no-provisioning-check', action='store_true',
279 help='Will not check if devices are provisioned properly.')
280 parser.add_option('--device-status-dashboard', action='store_true',
281 help='Output device status data for dashboard.')
282 parser.add_option('--restart-usb', action='store_true',
283 help='Restart USB ports before running device check.')
284 parser.add_option('--json-output',
285 help='Output JSON information into a specified file.')
287 options, args = parser.parse_args()
289 parser.error('Unknown options %s' % args)
291 # Remove the last build's "bad devices" before checking device statuses.
292 device_blacklist.ResetBlacklist()
295 expected_devices = device_list.GetPersistentDeviceList(
296 os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
298 expected_devices = []
299 devices = android_commands.GetAttachedDevices()
300 # Only restart usb if devices are missing.
301 if set(expected_devices) != set(devices):
302 print 'expected_devices: %s, devices: %s' % (expected_devices, devices)
306 if options.restart_usb:
308 usb_restarted = False
309 bb_annotations.PrintWarning()
310 print 'USB reset stage failed, wait for any device to come back.'
312 print 'retry adb devices...'
314 devices = android_commands.GetAttachedDevices()
315 if set(expected_devices) == set(devices):
316 # All devices are online, keep going.
318 if not usb_restarted and devices:
319 # The USB wasn't restarted, but there's at least one device online.
320 # No point in trying to wait for all devices.
324 # TODO(navabi): Test to make sure this fails and then fix call
325 offline_devices = android_commands.GetAttachedDevices(
326 hardware=False, emulator=False, offline=True)
328 types, builds, batteries, reports, errors = [], [], [], [], []
331 types, builds, batteries, reports, errors, fail_step_lst = (
332 zip(*[DeviceInfo(dev, options) for dev in devices]))
334 err_msg = CheckForMissingDevices(options, devices) or []
336 unique_types = list(set(types))
337 unique_builds = list(set(builds))
339 bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s'
340 % (len(devices), unique_types, unique_builds))
341 print '\n'.join(reports)
343 for serial, dev_errors in zip(devices, errors):
345 err_msg += ['%s errors:' % serial]
346 err_msg += [' %s' % error for error in dev_errors]
349 bb_annotations.PrintWarning()
350 msg = '\n'.join(err_msg)
352 from_address = 'buildbot@chromium.org'
353 to_addresses = ['chromium-android-device-alerts@google.com']
354 bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
355 slave_name = os.environ.get('BUILDBOT_SLAVENAME')
356 subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name)
357 SendEmail(from_address, to_addresses, [], subject, msg)
359 if options.device_status_dashboard:
360 perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
361 [len(devices)], 'devices')
362 perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
363 [len(offline_devices)], 'devices',
365 for serial, battery in zip(devices, batteries):
366 perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
370 if options.json_output:
371 with open(options.json_output, 'wb') as f:
373 'online_devices': devices,
374 'offline_devices': offline_devices,
375 'expected_devices': expected_devices,
376 'unique_types': unique_types,
377 'unique_builds': unique_builds,
380 if False in fail_step_lst:
381 # TODO(navabi): Build fails on device status check step if there exists any
382 # devices with critically low battery. Remove those devices from testing,
383 # allowing build to continue with good devices.
390 if __name__ == '__main__':