Implement --processes options for repa list

Repa list makes a lot of queries to OBS, which results in slow output.
With --processes it queries OBS in parallel utilizing functionality of
multiprocessing module.

This is an experimental feature. Hopefully it will go away when repa
will start using IRIS ReST API.

Fixes: #1762
Change-Id: I47424e5e5c044dd75589f0d0dcb01a7bbff068e8
Signed-off-by: Ed Bartosh <eduard.bartosh@intel.com>

diff --git a/repa.1 b/repa.1
index d7d522f16feec7751e7ef3dd8f0d54356b0f4789..656a1d5b0f0a444a367f1860af742d7554cc1a16 100644 (file)
--- a/repa.1
+++ b/repa.1
@@ -66,7 +66,7 @@ Print short help text and exit.
 .\"
 .\" The "list" command description
 .\"
-.SS \fBlist\fR [\-\-help] [\-\-regexp <regexp>]
+.SS \fBlist\fR [\-\-help] [\-\-regexp <regexp>] [\-\-processes <processes>]
 
 .RS 2
 List submissions in the following format:
@@ -119,6 +119,15 @@ Use REGEXP to filter out submission project. Regexp is applied to OBS project na
 Note, that regexp can also be specified in \fIrepa\fR configuration file.
 .RE
 
+.PP
+.B \-\-processes
+PROCESSES
+.RS 2
+Use PROCESSES to specify the amount of python processes to run in parallel. Usage of this option can significantly speed up repa list.
+Note, that this parameter can also be specified in \fIrepa\fR configuration file.
+
+.RE
+
 .\"
 .\" The "accept" command description
 .\"

diff --git a/repa/common.py b/repa/common.py
index 02ec1460bf06298e4d1290186ea21d8ba83d6d53..1f8c4305ae1c2551293255d9620931857be7bfcc 100644 (file)
--- a/repa/common.py
+++ b/repa/common.py
@@ -46,19 +46,19 @@ def get_project_by_name(obs, name, target):
                             (target, name))
     if len(projects) > 1:
 
-        plist = '\n  '.join(prj for prj, _ in projects)
+        plist = '\n  '.join(prj for prj, _desc, _build_results in projects)
         raise RepaException('%s %s resolves into multiple projects:\n  %s' % \
                             (target, name, plist))
 
-    return projects[0][0], json.loads(projects[0][1])
+    return projects[0][0], json.loads(projects[0][1]), projects[0][2]
 
 
 def _resolve_submissions(obs, name, target):
     """Get list of submissions with meta. Resolves submitgroups."""
-    project, meta = get_project_by_name(obs, name, target)
+    project, meta, _bresults = get_project_by_name(obs, name, target)
     if name.startswith('submitgroup'):
         for subm in meta['submissions']:
-            sprj, smeta = get_project_by_name(obs, subm, target)
+            sprj, smeta, _bresults = get_project_by_name(obs, subm, target)
             yield subm, sprj, smeta
     else:
         yield name, project, meta

diff --git a/repa/group.py b/repa/group.py
index 30c99666a749f85e4713d3a701070c25b6f14bdf..3b087b18c2d855359c165323dd8a21ace5048025 100755 (executable)
--- a/repa/group.py
+++ b/repa/group.py
@@ -127,14 +127,16 @@ def group_submissions(obs, submissions, target, comment, force=False):
     # find correspondent prerelease projects
     info = {}
     for submission in submissions:
-        project, meta = get_project_by_name(obs, submission, target)
-        info[submission] = {'project': project, 'meta': meta}
+        project, meta, build_results = get_project_by_name(obs, submission,
+                                                           target)
+        info[submission] = {'project': project, 'meta': meta,
+                            'build_results': build_results}
 
     # Validate submissions
     check_target_prj(info)
 
-    bresults = [(subm, data['project'], obs.get_build_results(data['project']))\
-                 for subm, data in info.iteritems()]
+    bresults = [(subm, data['project'], data['build_results']) \
+                     for subm, data in info.iteritems()]
     check_build_results(bresults)
 
     check_binary_pkgs(obs, info, force)

diff --git a/repa/list.py b/repa/list.py
index 7e720ef29efbd5921b4ee151b3edda0ba5b0ae66..37e8d5d5d942f10ff6f51fe913129d8f1af93bbd 100755 (executable)
--- a/repa/list.py
+++ b/repa/list.py
@@ -38,11 +38,11 @@ from repa.obs import OBS
 from repa.main import sub_main
 
 
-def get_status(obs, project):
+def get_status(results):
     """Gest submission status."""
     # Process project build results
     codes = set()
-    for target in obs.get_build_results(project).itervalues():
+    for target in results.itervalues():
         codes.add(target.get('code'))
         codes.add(target.get('state'))
         for pkginfo in target['packages']:
@@ -82,12 +82,13 @@ def get_obs_url(meta, buildurl='https://build.tizen.org'):
                         % (meta['obs_target_prj'], name.replace('/', ':')))
 
 
-def list_submissions(obs, regexp):
+def list_submissions(obs, regexp, processes):
     """List submissions and groups."""
     # submissions
     groups = []
-    for project, desc in obs.get_projects('^%s.*%s' %
-                                          (OBS_PROJECT_PREFIX, regexp)):
+    for project, desc, build_results in \
+                obs.get_projects('^%s.*%s' % (OBS_PROJECT_PREFIX, regexp),
+                                 processes):
         meta = json.loads(desc)
         if ':submitgroup:' in project:
             groups.append(meta)
@@ -100,7 +101,7 @@ def list_submissions(obs, regexp):
         obs_url = get_obs_url(meta)
         if obs_url:
             print '    obs url: ', obs_url
-        print "    builds:", get_status(obs, project)
+        print "    builds:", get_status(build_results)
         show_images(meta)
         print
 
@@ -135,12 +136,15 @@ class List(object):
         """
         parser.add_argument('-r', '--regexp', help='search regexp',
                             default=config.get('list_regexp', '.*'))
+        parser.add_argument('--processes', type=int,
+                            help='amount of parallel processes to use',
+                            default=config.get('processes'))
 
     @staticmethod
     def run(argv):
         """Command line entry point. Called from [sub_]main"""
         obs = OBS(argv.apiurl, argv.apiuser, argv.apipasswd)
-        return list_submissions(obs, argv.regexp)
+        return list_submissions(obs, argv.regexp, argv.processes)
 
 
 if __name__ == '__main__':

diff --git a/repa/obs.py b/repa/obs.py
index 4941cbf1837c27daf6c4be4f5c4d4de9ba319c67..96f0625c273ba4313d9bc322618d9c48a7e2a002 100644 (file)
--- a/repa/obs.py
+++ b/repa/obs.py
@@ -77,7 +77,7 @@ class OBS(OSC):
         OSC.__init__(self, apiurl, self.oscrcpath)
 
 
-    def get_projects(self, regexp=''):
+    def get_projects(self, regexp='', processes=0):
         """List projects with attributes."""
         try:
             projects = core.meta_get_project_list(self.apiurl)
@@ -85,14 +85,25 @@ class OBS(OSC):
             raise RepaException("cat't get list of projects from %s: %s" %
                                 (self.apiurl, err))
 
+        if processes > 1:
+            from multiprocessing.pool import ThreadPool
+            pool = ThreadPool(processes=processes)
+            processes = {}
+            for project in projects:
+                if regexp and re.match(regexp, project):
+                    processes[project] = (
+                        pool.apply_async(self.get_description, [project]),
+                        pool.apply_async(self.get_build_results, [project]))
+
         for project in projects:
-            if regexp and not re.match(regexp, project):
-                continue
-            try:
-                yield project, self.get_description(project)
-            except OSCError as err:
-                raise RepaException("Can't get a description from %s: %s" %
-                                    (project, err))
+            if regexp and re.match(regexp, project):
+                if processes > 1:
+                    yield (project, processes[project][0].get(),
+                                    processes[project][1].get())
+                else:
+                    yield (project, self.get_description(project),
+                                    self.get_build_results(project))
+
 
     def get_build_results(self, prj):
         """Get project build results."""