Fixed processing of architectures in repomaker

Repomaker was getting list of architectures analyzinng packages in the
incoming repository. If repository doesn't contain some architectures
those were skipped. This was a mistake. Now repomaker gets list of
architectures as a parameter and creates output repos for all
architectures even if no packages found in incoming repo for some of
them.

Fixes: #1324

Change-Id: I2e4fa4df61ae1fcaa64d0792648391493c962001
Signed-off-by: Ed Bartosh <eduard.bartosh@intel.com>

diff --git a/common/repomaker.py b/common/repomaker.py
index 4c8ee69..387a2dc 100644 (file)
--- a/common/repomaker.py
+++ b/common/repomaker.py
@@ -56,7 +56,7 @@ def collect(in_dir):
         ftype = ARCH_MAP.get(ftype, ftype)
 
         basename = os.path.basename(fname)
-        if ftype != "src":
+        if ftype not in ("src", "noarch"):
             archs.add(ftype)
 
         is_debug = "-debugsource-" in fname or "-debuginfo-" in fname
@@ -122,8 +122,8 @@ class RepoMaker(object):
         self.repos = {}
         self.imagedata = None
 
-    def add_repo(self, in_dir, name, buildconf=None, move=False, gpg_key=None,
-                 signer='/usr/bin/sign'):
+    def add_repo(self, in_dir, name, archs = (), buildconf=None,
+                 move=False, gpg_key=None, signer='/usr/bin/sign'):
         """
         Convert repository to download structure.
         Create or update repository using packages from repo_dir
@@ -131,6 +131,7 @@ class RepoMaker(object):
         Args:
             in_dir (str):  path to repository to convert
             name (str):  name of the repository to create
+            archs (tuple): list of architectures
             buildconf(str): content of build configuration
             move (bool): move files instead of hardlinking
             gpg_key (str): path to file with gpg key
@@ -144,7 +145,7 @@ class RepoMaker(object):
 
         repo_dir = os.path.join(self.outdir, "repos", name)
         if name not in self.repos:
-            self.repos[name] = {'archs': set()}
+            self.repos[name] = {'archs': set(archs)}
 
         files, archs = collect(in_dir)
         self.repos[name]['archs'].update(archs)
@@ -154,7 +155,8 @@ class RepoMaker(object):
 
         for fpath, ftype, is_debug, is_group, is_imageconf in files:
             # Prepare list of target directories
-            target_dirs = gen_target_dirs(repo_dir, archs, ftype, is_debug)
+            target_dirs = gen_target_dirs(repo_dir, self.repos[name]['archs'],
+                                          ftype, is_debug)
 
             # Move or hardlink .rpm to target dirs
             move_or_hardlink(fpath, target_dirs, move)

diff --git a/job_pre_release_obs.py b/job_pre_release_obs.py
index 3cbe1c7..6f6302b 100755 (executable)
--- a/job_pre_release_obs.py
+++ b/job_pre_release_obs.py
@@ -79,7 +79,8 @@ def get_prerelease_data(backenddb, base_url, base_path='/srv/download',
             'prerelease_dir': prerelease_dir,
             'prerelease_path': os.path.join(base_path, prerelease_dir),
             'prerelease_build_id': build_id,
-            'pkg_urls': pkg_urls
+            'pkg_urls': pkg_urls,
+            'archs': repo['Architectures']
             }
 
 
@@ -174,7 +175,7 @@ def make_repo(project, repo, backenddb, base_url, base_path, live_repo_base):
 
     # TODO: get buldconf from OBS
     try:
-        repomaker.add_repo(live_repo_path, repo, move=False) #,buildconf
+        repomaker.add_repo(live_repo_path, repo, data['archs'], move=False)
     except RepoMakerError, err:
         raise LocalError("Unable to create download repo: %s" % err)
 

diff --git a/tests/test_repomaker.py b/tests/test_repomaker.py
index b841d9e..048fc86 100644 (file)
--- a/tests/test_repomaker.py
+++ b/tests/test_repomaker.py
@@ -78,6 +78,21 @@ def tempdir(suffix='', prefix='tmp'):
     finally:
         shutil.rmtree(tmpdir)
 
+def prepare_dir(in_repo, arches):
+    """Create test directory structure."""
+    files = ["%s-%s.%s.rpm" % (name, version, arch) \
+             for name in ("pkg1", "pkg2", "pkg3") \
+             for version in ("0.1-2.3", "1.2-3.4") \
+             for arch in arches]
+    for fname in files:
+        subdir = fname.split('.')[-2]
+        #subdir = ('', fname)[files.index(fname) % 2]
+        dir_path = os.path.join(in_repo, subdir)
+        if not os.path.exists(dir_path):
+            os.makedirs(dir_path)
+        with open(os.path.join(dir_path, fname), 'w') as frpm:
+            frpm.write(RPM)
+
 
 class RepoMakerTest(unittest.TestCase):
     '''Tests for RepoMaker functionality.'''
@@ -91,24 +106,12 @@ class RepoMakerTest(unittest.TestCase):
     def test_add_repo(self):
         """Test converting repository to the download structure."""
         with tempdir(prefix='repomaker.', suffix='.inrepo') as in_repo:
-            # prepare directory with rpms
-            arches = ('src', 'noarch', 'i586', 'x86_64')
-            files = ["%s-%s.%s.rpm" % (name, version, arch) \
-                     for name in ("pkg1", "pkg2", "pkg3") \
-                     for version in ("0.1-2.3", "1.2-3.4") \
-                     for arch in arches]
-            for fname in files:
-                subdir = ('', fname)[files.index(fname)%2]
-                dir_path = os.path.join(in_repo, subdir)
-                if not os.path.exists(dir_path):
-                    os.makedirs(dir_path)
-                with open(os.path.join(dir_path, fname), 'w') as frpm:
-                    frpm.write(RPM)
-
+            prepare_dir(in_repo, ['src', 'noarch', 'i586', 'x86_64'])
             with tempdir(prefix='repomaker.', suffix='.outrepo') as out_repo:
                 maker = RepoMaker('test-id', out_repo)
-                maker.add_repo(in_repo, 'testrepo', move=True, gpg_key='key',
-                               signer='/bin/true', buildconf='buildconf')
+                maker.add_repo(in_repo, 'testrepo', ('i586', 'x86_64'),
+                               move=True, gpg_key='key', signer='/bin/true',
+                               buildconf='buildconf')
                 # Check repo structure
                 results = {
                     'sources':
@@ -146,3 +149,57 @@ class RepoMakerTest(unittest.TestCase):
                     packages = sorted(elem.getAttribute('href') for elem in \
                                           dom.getElementsByTagName('location'))
                     self.assertEqual(packages, results[arch])
+
+
+    def test_add_repo_noarch(self):
+        """
+        Test converting repository to the download structure.
+        Input repo contains only noarch packages.
+        """
+        with tempdir(prefix='repomaker.', suffix='.inrepo') as in_repo:
+            prepare_dir(in_repo, ['src', 'noarch'])
+
+            with tempdir(prefix='repomaker.', suffix='.outrepo') as out_repo:
+                maker = RepoMaker('test-id', out_repo)
+                maker.add_repo(in_repo, 'testrepo-noarch', ('ia32', 'x86_64'),
+                               move=True, gpg_key='key', signer='/bin/true',
+                               buildconf='buildconf')
+                # Check repo structure
+                results = {
+                    'sources':
+                        ['pkg1-0.1-2.3.src.rpm', 'pkg1-1.2-3.4.src.rpm',
+                         'pkg2-0.1-2.3.src.rpm', 'pkg2-1.2-3.4.src.rpm',
+                         'pkg3-0.1-2.3.src.rpm', 'pkg3-1.2-3.4.src.rpm'],
+                    'ia32/packages':
+                        ['pkg1-0.1-2.3.noarch.rpm',
+                         'pkg1-1.2-3.4.noarch.rpm',
+                         'pkg2-0.1-2.3.noarch.rpm',
+                         'pkg2-1.2-3.4.noarch.rpm',
+                         'pkg3-0.1-2.3.noarch.rpm',
+                         'pkg3-1.2-3.4.noarch.rpm'],
+                    'ia32/debug': [],
+                    'x86_64/packages':
+                        ['pkg1-0.1-2.3.noarch.rpm',
+                         'pkg1-1.2-3.4.noarch.rpm',
+                         'pkg2-0.1-2.3.noarch.rpm',
+                         'pkg2-1.2-3.4.noarch.rpm',
+                         'pkg3-0.1-2.3.noarch.rpm',
+                         'pkg3-1.2-3.4.noarch.rpm'],
+                    'x86_64/debug': []
+                }
+
+                for arch in ('sources', 'ia32/packages', 'ia32/debug',
+                                        'x86_64/packages', 'x86_64/debug'):
+                    primary_path = glob.glob(os.path.join(out_repo,
+                                                          'test-id',
+                                                          'repos',
+                                                          'testrepo-noarch',
+                                                          arch,
+                                                          'repodata',
+                                                          '*primary.xml.gz'))[0]
+                    #print 'primary_path=', primary_path
+                    dom = minidom.parse(gzip.open(primary_path))
+                    packages = sorted(elem.getAttribute('href') for elem in \
+                                          dom.getElementsByTagName('location'))
+                    print 'packages:', arch, packages, results[arch]
+                    self.assertEqual(packages, results[arch])