2 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Dromaeo benchmark automation script.
8 Script runs dromaeo tests in browsers specified by --browser switch and saves
9 results to a spreadsheet on docs.google.com.
12 1. Install Google Data APIs Python Client Library from
13 http://code.google.com/p/gdata-python-client.
14 2. Checkout Dromaeo benchmark from
15 http://src.chromium.org/svn/trunk/src/chrome/test/data/dromaeo and provide
16 local path to it in --dromaeo_home switch.
17 3. Create a spreadsheet at http://docs.google.com and specify its name in
20 Benchmark results are presented in the following format:
22 test 1 name|m11|...|m1n|test 1 average mean| |e11|...|e1n|test 1 average error
23 test 2 name|m21|...|m2n|test 2 average mean| |e21|...|e2n|test 2 average error
26 Here mij is mean run/s in individual dromaeo test i during benchmark run j,
27 eij is error in individual dromaeo test i during benchmark run j.
30 dromaeo_benchmark_runner.py -b "E:\chromium\src\chrome\Release\chrome.exe"
31 -b "C:\Program Files (x86)\Safari\safari.exe"
32 -b "C:\Program Files (x86)\Opera 10.50 pre-alpha\opera.exe" -n 1
33 -d "E:\chromium\src\chrome\test\data\dromaeo" -f dom -e example@gmail.com
44 from optparse import OptionParser
45 from BaseHTTPServer import HTTPServer
46 import SimpleHTTPServer
47 import gdata.spreadsheet.service
49 max_spreadsheet_columns = 20
50 test_props = ['mean', 'error']
54 parser = OptionParser()
55 parser.add_option("-b", "--browser",
56 action="append", dest="browsers",
57 help="list of browsers to test")
58 parser.add_option("-n", "--run_count", dest="run_count", type="int",
59 default=5, help="number of runs")
60 parser.add_option("-d", "--dromaeo_home", dest="dromaeo_home",
61 help="directory with your dromaeo files")
62 parser.add_option("-p", "--port", dest="port", type="int",
63 default=8080, help="http server port")
64 parser.add_option("-f", "--filter", dest="filter",
65 default="dom", help="dromaeo suite filter")
66 parser.add_option("-e", "--email", dest="email",
67 help="your google docs account")
68 parser.add_option("-s", "--spreadsheet", dest="spreadsheet_title",
70 help="your google docs spreadsheet name")
72 options = parser.parse_args()[0]
74 if not options.dromaeo_home:
75 raise Exception('please specify dromaeo_home')
80 def KillProcessByName(process_name):
81 process = subprocess.Popen('wmic process get processid, executablepath',
82 stdout=subprocess.PIPE)
83 stdout = str(process.communicate()[0])
84 match = re.search(re.escape(process_name) + '\s+(\d+)', stdout)
87 subprocess.call('taskkill /pid %s' % pid)
90 class SpreadsheetWriter(object):
91 "Utility class for storing benchmarking results in Google spreadsheets."
93 def __init__(self, email, spreadsheet_title):
94 '''Login to google docs and search for spreadsheet'''
96 self.token_file = os.path.expanduser("~/.dromaeo_bot_auth_token")
97 self.gd_client = gdata.spreadsheet.service.SpreadsheetsService()
100 if os.path.exists(self.token_file):
103 file = open(self.token_file, 'r')
106 self.gd_client.SetClientLoginToken(token)
107 self.gd_client.GetSpreadsheetsFeed()
109 except (IOError, gdata.service.RequestError):
111 if not authenticated:
112 self.gd_client.email = email
113 self.gd_client.password = getpass.getpass('Password for %s: ' % email)
114 self.gd_client.source = 'python robot for dromaeo'
115 self.gd_client.ProgrammaticLogin()
116 token = self.gd_client.GetClientLoginToken()
118 file = open(self.token_file, 'w')
123 os.chmod(self.token_file, 0600)
125 # Search for the spreadsheet with title = spreadsheet_title.
126 spreadsheet_feed = self.gd_client.GetSpreadsheetsFeed()
127 for spreadsheet in spreadsheet_feed.entry:
128 if spreadsheet.title.text == spreadsheet_title:
129 self.spreadsheet_key = spreadsheet.id.text.rsplit('/', 1)[1]
130 if not self.spreadsheet_key:
131 raise Exception('Spreadsheet %s not found' % spreadsheet_title)
133 # Get the key of the first worksheet in spreadsheet.
134 worksheet_feed = self.gd_client.GetWorksheetsFeed(self.spreadsheet_key)
135 self.worksheet_key = worksheet_feed.entry[0].id.text.rsplit('/', 1)[1]
137 def _InsertRow(self, row):
138 row = dict([('c' + str(i), row[i]) for i in xrange(len(row))])
139 self.gd_client.InsertRow(row, self.spreadsheet_key, self.worksheet_key)
141 def _InsertBlankRow(self):
142 self._InsertRow('-' * self.columns_count)
144 def PrepareSpreadsheet(self, run_count):
145 """Update cells in worksheet topmost row with service information.
147 Calculate column count corresponding to run_count and create worksheet
148 column titles [c0, c1, ...] in the topmost row to speed up spreadsheet
149 updates (it allows to insert a whole row with a single request)
152 # Calculate the number of columns we need to present all test results.
153 self.columns_count = (run_count + 2) * len(test_props)
154 if self.columns_count > max_spreadsheet_columns:
155 # Google spreadsheet has just max_spreadsheet_columns columns.
156 max_run_count = max_spreadsheet_columns / len(test_props) - 2
157 raise Exception('maximum run count is %i' % max_run_count)
158 # Create worksheet column titles [c0, c1, ..., cn].
159 for i in xrange(self.columns_count):
160 self.gd_client.UpdateCell(1, i + 1, 'c' + str(i), self.spreadsheet_key,
163 def WriteColumnTitles(self, run_count):
164 "Create titles for test results (mean 1, mean 2, ..., average mean, ...)"
166 for prop in test_props:
168 for i in xrange(run_count):
169 row.append('%s %i' % (prop, i + 1))
170 row.append('average ' + prop)
173 def WriteBrowserBenchmarkTitle(self, browser_name):
174 "Create browser benchmark title (browser name, date time)"
175 self._InsertBlankRow()
176 self._InsertRow([browser_name, time.strftime('%d.%m.%Y %H:%M:%S')])
178 def WriteBrowserBenchmarkResults(self, test_name, test_data):
179 "Insert a row with single test results"
181 for prop in test_props:
183 row.append(test_name)
186 row.extend([str(x) for x in test_data[prop]])
187 row.append(str(sum(test_data[prop]) / len(test_data[prop])))
191 class DromaeoHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
194 self.send_response(200)
196 self.wfile.write("<HTML>POST OK.<BR><BR>");
197 length = int(self.headers.getheader('content-length'))
198 parameters = urlparse.parse_qs(self.rfile.read(length))
199 self.server.got_post = True
200 self.server.post_data = parameters['data']
203 class BenchmarkResults(object):
204 "Storage class for dromaeo benchmark results"
209 def ProcessBrowserPostData(self, data):
210 "Convert dromaeo test results in internal format"
211 tests = json.loads(data[0])
213 test_name = test['name']
214 if test_name not in self.data:
215 # Test is encountered for the first time.
216 self.data[test_name] = dict([(prop, []) for prop in test_props])
217 # Append current run results.
218 for prop in test_props:
220 if prop in test: value = test[prop] # workaround for Opera 10.5
221 self.data[test_name][prop].append(value)
225 options = ParseArguments()
227 # Start sever with dromaeo.
228 os.chdir(options.dromaeo_home)
229 server = HTTPServer(('', options.port), DromaeoHandler)
231 # Open and prepare spreadsheet on google docs.
232 spreadsheet_writer = SpreadsheetWriter(options.email,
233 options.spreadsheet_title)
234 spreadsheet_writer.PrepareSpreadsheet(options.run_count)
235 spreadsheet_writer.WriteColumnTitles(options.run_count)
237 for browser in options.browsers:
238 browser_name = os.path.splitext(os.path.basename(browser))[0]
239 spreadsheet_writer.WriteBrowserBenchmarkTitle(browser_name)
240 benchmark_results = BenchmarkResults()
241 for run_number in xrange(options.run_count):
242 print '%s run %i' % (browser_name, run_number + 1)
244 test_page = 'http://localhost:%i/index.html?%s&automated&post_json' % (
245 options.port, options.filter)
246 browser_process = subprocess.Popen('%s "%s"' % (browser, test_page))
247 server.got_post = False
248 server.post_data = None
249 # Wait until POST request from browser.
250 while not server.got_post:
251 server.handle_request()
252 benchmark_results.ProcessBrowserPostData(server.post_data)
254 KillProcessByName(browser)
255 browser_process.wait()
257 # Insert test results into spreadsheet.
258 for (test_name, test_data) in benchmark_results.data.iteritems():
259 spreadsheet_writer.WriteBrowserBenchmarkResults(test_name, test_data)
261 server.socket.close()
265 if __name__ == '__main__':