Draft implementation of grouping

Grouping is done by creating group project and aggregating binaries
from submission projects into it.

Various checks for submissions have been implemented. Submissions
should have been published, they should not have common packages and
all grouped projects should have the same target project.

Note, that backend should be adjusted to build images for group
projects the same way as for submission projects.

Fixes: #1143

Change-Id: I5bdff34c951453707579579620bf3c28a0417d21
Signed-off-by: Ed Bartosh <eduard.bartosh@intel.com>
Reviewed-on: https://otctools.jf.intel.com/review/5900
Tested-by: OTC Tools Tester <ed.bartosh@linux.intel.com>

diff --git a/repa/group.py b/repa/group.py
index 0c3089a..95b71e8 100755 (executable)
--- a/repa/group.py
+++ b/repa/group.py
@@ -12,8 +12,124 @@ Group submissions.
 """
 
 import sys
+import time
+import json
+
+from collections import defaultdict
+from StringIO import StringIO
 
 from repa.main import sub_main
+from repa.obs import OBS, OBS_PREFIX
+from repa.common import RepaException
+
+
+def check_target_prj(submissions):
+    """Check if target projects are the same for all submissions"""
+    result = defaultdict(list)
+    for submission, data in submissions.iteritems():
+        result[data['meta']['obs_target_prj']].append(submission)
+    if len(result) > 1:
+        message = '\n'.join('%s: %s' % (project, ' '.join(subms)) \
+                                for project, subms in result.iteritems())
+        raise RepaException('Target projects differ:\n%s\n' % message)
+
+
+def check_build_results(bresults):
+    """Check if build targets are published."""
+    for subm, _prj, results in bresults:
+        for target, res in results.iteritems():
+            if res['state'] != 'published' or res['code'] != 'published':
+                if res['packages']:
+                    raise RepaException("%s: target %s is not published yet" %
+                                        (subm, target))
+            # check if status of package builds is not worse than in
+            # target project: for pkg, status in res['packages'] ...
+
+
+def check_common_pkgs(obs, submissions):
+    """Check if submissions have common packages"""
+    binaries = {}
+    for submission, data in submissions.iteritems():
+        pkgs = list(obs.get_binary_packages(data['project']))
+        # check if submission has binary packages
+        for repo, bins in pkgs:
+            #if not bins:
+            #    raise RepaException('No binary packages found in %s %s/%s' % \
+            #                            (submission, repo[0], repo[1]))
+            # check if submissions have common packages
+            for subm, info in binaries.iteritems():
+                if repo == info['repo']:
+                    common = set(info['bins']).intersection(bins)
+                    if common:
+                        msg = '%s and %s have common packages: %s' % \
+                              (subm, submission, ','.join(common))
+                        raise RepaException(msg)
+
+
+def create_group_project(obs, submissions, meta, comment):
+    """Create group project."""
+    target_prj = str(meta['obs_target_prj'])
+    timestamp = time.strftime('%Y%m%d.%H%M%S', time.gmtime())
+    name = "submitgroup/%s/%s" % ('/'.join(meta['git_tag'].split('/')[1:-1]),
+                                  timestamp)
+    gmeta = {'name': name, 'obs_target_prj': target_prj,
+             'submissions': submissions, 'comment': comment}
+    project = '%s%s' % (OBS_PREFIX, name.replace('/', ':'))
+
+    saved = sys.stdout
+    sys.stdout = StringIO()
+    try:
+        obs.create_project(project, target_prj, description=json.dumps(gmeta))
+    finally:
+        sys.stdout = saved
+    return name, str(project)
+
+
+def aggregate(obs, bresults, gproject):
+    """Aggregate packages into group project."""
+    aggregated = set()
+    for subm, prj, results in bresults:
+        for res in results.itervalues():
+            for pkg, state in res['packages']:
+                if state == 'succeeded' and pkg not in aggregated:
+                    print 'aggregating %s/%s' % (subm, pkg)
+                    obs.aggregate_package(prj, pkg, gproject, pkg)
+                    aggregated.add(pkg)
+
+    return aggregated
+
+
+def group_submissions(obs, submissions, comment):
+    """Group multiple submissions into one group."""
+    # find correspondent prerelease projects
+    info = {}
+    for submission in submissions:
+        suffix = submission.replace('/', ':')
+        found = list(obs.get_projects('^%s.*:%s$' % (OBS_PREFIX, suffix)))
+        if not found:
+            raise RepaException('OBS project not found for %s' % submission)
+        project, description = found[0]
+        info[submission] = {'project': project, 'meta': json.loads(description)}
+
+    # Validate submissions
+    check_target_prj(info)
+
+    bresults = [(subm, data['project'], obs.get_build_results(data['project']))\
+                 for subm, data in info.iteritems()]
+    check_build_results(bresults)
+
+    check_common_pkgs(obs, info)
+
+    # create group project
+    name, gproject = create_group_project(obs, info.keys(),
+                                          info.itervalues().next()['meta'],
+                                          comment)
+    print 'Created submit group %s\n' % name
+
+    aggregated = aggregate(obs, bresults, gproject)
+
+    print '\n%d submissions (%d packages) have been merged into %s' % \
+          (len(info), len(aggregated), name)
 
 class Group(object):
     """Subcommand: Manage group  submissions."""
@@ -25,12 +141,15 @@ class Group(object):
     @staticmethod
     def add_arguments(parser, _config):
         """Adds arguments to the parser. Called from [sub_]main."""
-        parser.add_argument('--comment', help='comment')
+        parser.add_argument('submission', nargs='+',
+                           help='space separated list of submissions')
+        parser.add_argument('-comment', help='comment', default='')
 
-    def run(self, argv):
+    @staticmethod
+    def run(argv):
         """Command line entry point. Called from [sub_]main."""
-        print '%s: Not implemented yet' % self.help
-        print 'comment=%s' % argv.comment
+        obs = OBS(argv.apiurl, argv.apiuser, argv.apipasswd)
+        return group_submissions(obs, argv.submission, argv.comment)
 
 
 if __name__ == '__main__':