Make gcmole execute in parallel.

TBR=tandrii@chromium.org
NOTRY=true

Review URL: https://codereview.chromium.org/931233002

Cr-Commit-Position: refs/heads/master@{#26724}

diff --git a/tools/gcmole/gcmole.lua b/tools/gcmole/gcmole.lua
index 68dc491..9739684 100644 (file)
--- a/tools/gcmole/gcmole.lua
+++ b/tools/gcmole/gcmole.lua
@@ -34,6 +34,9 @@ local FLAGS = {
    -- Do not build gcsuspects file and reuse previously generated one.
    reuse_gcsuspects = false;
 
+   -- Don't use parallel python runner.
+   sequential = false;
+
    -- Print commands to console before executing them.
    verbose = false;
 
@@ -113,20 +116,66 @@ local function MakeClangCommandLine(
       .. " " .. arch_options
 end
 
+local function IterTable(t)
+  return coroutine.wrap(function ()
+    for i, v in ipairs(t) do
+      coroutine.yield(v)
+    end
+  end)
+end
+
+local function SplitResults(lines, func)
+   -- Splits the output of parallel.py and calls func on each result.
+   -- Bails out in case of an error in one of the executions.
+   local current = {}
+   local filename = ""
+   for line in lines do
+      local new_file = line:match "^______________ (.*)$"
+      local code = line:match "^______________ finish (%d+) ______________$"
+      if code then
+         if tonumber(code) > 0 then
+            log(table.concat(current, "\n"))
+            log("Failed to examine " .. filename)
+            return false
+         end
+         log("-- %s", filename)
+         func(filename, IterTable(current))
+      elseif new_file then
+         filename = new_file
+         current = {}
+      else
+         table.insert(current, line)
+      end
+   end
+   return true
+end
+
 function InvokeClangPluginForEachFile(filenames, cfg, func)
    local cmd_line = MakeClangCommandLine(cfg.plugin,
                                          cfg.plugin_args,
                                          cfg.triple,
                                          cfg.arch_define,
                                          cfg.arch_options)
-   for _, filename in ipairs(filenames) do
-      log("-- %s", filename)
-      local action = cmd_line .. " " .. filename .. " 2>&1"
+   if FLAGS.sequential then
+      log("** Sequential execution.")
+      for _, filename in ipairs(filenames) do
+         log("-- %s", filename)
+         local action = cmd_line .. " " .. filename .. " 2>&1"
+         if FLAGS.verbose then print('popen ', action) end
+         local pipe = io.popen(action)
+         func(filename, pipe:lines())
+         local success = pipe:close()
+         if not success then error("Failed to run: " .. action) end
+      end
+   else
+      log("** Parallel execution.")
+      local action = "python tools/gcmole/parallel.py \""
+         .. cmd_line .. "\" " .. table.concat(filenames, " ")
       if FLAGS.verbose then print('popen ', action) end
       local pipe = io.popen(action)
-      func(filename, pipe:lines())
-      local success = pipe:close()
-      if not success then error("Failed to run: " .. action) end
+      local success = SplitResults(pipe:lines(), func)
+      local closed = pipe:close()
+      if not (success and closed) then error("Failed to run: " .. action) end
    end
 end
 

diff --git a/tools/gcmole/parallel.py b/tools/gcmole/parallel.py
new file mode 100755 (executable)
index 0000000..0c045f4
--- /dev/null
+++ b/tools/gcmole/parallel.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# Copyright 2015 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+This script calls the first argument for each of the following arguments in
+parallel. E.g.
+parallel.py "clang --opt" file1 file2
+calls
+clang --opt file1
+clang --opt file2
+
+The output (stdout and stderr) is concatenated sequentially in the form:
+______________ file1
+<output of clang --opt file1>
+______________ finish <exit code of clang --opt file1> ______________
+______________ file2
+<output of clang --opt file2>
+______________ finish <exit code of clang --opt file2> ______________
+"""
+
+import itertools
+import multiprocessing
+import subprocess
+import sys
+
+def invoke(cmdline):
+  try:
+    return (subprocess.check_output(
+        cmdline, shell=True, stderr=subprocess.STDOUT), 0)
+  except subprocess.CalledProcessError as e:
+    return (e.output, e.returncode)
+
+if __name__ == '__main__':
+  assert len(sys.argv) > 2
+  processes = multiprocessing.cpu_count()
+  pool = multiprocessing.Pool(processes=processes)
+  cmdlines = ["%s %s" % (sys.argv[1], filename) for filename in sys.argv[2:]]
+  for filename, result in itertools.izip(
+      sys.argv[2:], pool.imap(invoke, cmdlines)):
+    print "______________ %s" % filename
+    print result[0]
+    print "______________ finish %d ______________" % result[1]