obs/get_projects: retry OBS operations

get_projects starts to fail to query OBS more frequently in parallel
mode. Using nice implementation of retry decorator to repeat queries for
2 methods running under multiprocessing pool.

Change-Id: I175a528d9c0d7aa2951c3b659a1f5f85a8ae614f
Signed-off-by: Ed Bartosh <eduard.bartosh@intel.com>

diff --git a/repa/common.py b/repa/common.py
index 1f8c4305ae1c2551293255d9620931857be7bfcc..2a21d090a9cacf7b4b637b53d62a66c30e59e2c4 100644 (file)
--- a/repa/common.py
+++ b/repa/common.py
@@ -29,7 +29,10 @@ Common module.
 Common functions, classes, exceptions.
 """
 
+import sys
+import time
 import json
+from functools import wraps
 
 OBS_PROJECT_PREFIX = "home:prerelease:"
 
@@ -119,3 +122,30 @@ def accept_or_reject(obs, submission, state, target, comment=''):
     if submission.startswith('submitgroup'):
         delete_project(obs, submission, target)
 
+class CancelRetryError(Exception):
+    """Exception for handling cancelling of the retry loop. Needed for
+    transparently re-raising the previous exception."""
+    def __init__(self):
+        Exception.__init__(self)
+        self.typ, self.val, self.backtrace = sys.exc_info()
+
+def retry(exceptions, tries=10, sleep=1):
+    """Decorator for re-trying function calls"""
+    def decorator(func):
+        """The "real" decorator function"""
+        @wraps(func)
+        def wrap(*args, **kwargs):
+            """Wrapper for re-trying func"""
+            for attempt in range(1, tries + 1):
+                try:
+                    return func(*args, **kwargs)
+                except CancelRetryError as err:
+                    raise err.typ, err.val, err.backtrace
+                except exceptions as err:
+                    if attempt >= tries:
+                        raise
+                    elif sleep:
+                        time.sleep(sleep)
+        return wrap
+    return decorator
+

diff --git a/repa/obs.py b/repa/obs.py
index 96f0625c273ba4313d9bc322618d9c48a7e2a002..62277345e60652e1e36653b0606ce68426f5294c 100644 (file)
--- a/repa/obs.py
+++ b/repa/obs.py
@@ -39,13 +39,14 @@ import re
 from base64 import b64encode
 from xml.etree import cElementTree as ET
 from StringIO import StringIO
+from urllib2 import HTTPError
 
 from osc import core
 
 from gitbuildsys.oscapi import OSC, OSCError
 from gitbuildsys.utils import Temp
 
-from repa.common import RepaException
+from repa.common import RepaException, retry
 
 
 OSCRC_TEMPLATE = """[general]
@@ -77,6 +78,11 @@ class OBS(OSC):
         OSC.__init__(self, apiurl, self.oscrcpath)
 
 
+    @retry((OSCError, HTTPError))
+    def get_descr(self, project):
+        """Wrapper around get_description to be able to use @retry."""
+        return self.get_description(project)
+
     def get_projects(self, regexp='', processes=0):
         """List projects with attributes."""
         try:
@@ -92,7 +98,7 @@ class OBS(OSC):
             for project in projects:
                 if regexp and re.match(regexp, project):
                     processes[project] = (
-                        pool.apply_async(self.get_description, [project]),
+                        pool.apply_async(self.get_descr, [project]),
                         pool.apply_async(self.get_build_results, [project]))
 
         for project in projects:
@@ -101,10 +107,11 @@ class OBS(OSC):
                     yield (project, processes[project][0].get(),
                                     processes[project][1].get())
                 else:
-                    yield (project, self.get_description(project),
+                    yield (project, self.get_descr(project),
                                     self.get_build_results(project))
 
 
+    @retry((OSCError, HTTPError))
     def get_build_results(self, prj):
         """Get project build results."""
         meta = core.show_prj_results_meta(self.apiurl, prj)