From: Seth Vidal Date: Wed, 20 Feb 2008 20:22:49 +0000 (-0500) Subject: allow --pkglist or self.conf.pkglist to be a list of arbitrary X-Git-Tag: upstream/0.9.9~125 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=bb9a6387107b3ecd356ab995079735fe15bc8ad5;p=tools%2Fcreaterepo.git allow --pkglist or self.conf.pkglist to be a list of arbitrary urls to packages. createrepo will download the remote urls to a tempdir, read them in and add them to the metadata. --- diff --git a/createrepo/__init__.py b/createrepo/__init__.py index 2e5fa6e..e91cf85 100644 --- a/createrepo/__init__.py +++ b/createrepo/__init__.py @@ -21,6 +21,8 @@ import fnmatch import time import yumbased import shutil +from urlgrabber import grabber +import tempfile from yum import misc, Errors import rpmUtils.transaction @@ -72,6 +74,7 @@ class MetaDataConfig(object): self.directories = [] self.changelog_limit = None # needs to be an int or None self.unique_md_filenames = False + class SimpleMDCallBack(object): @@ -278,8 +281,6 @@ class MetaDataGenerator: # function for the first dir passed to --split, not all of them # this needs to be fixed by some magic in readMetadata.py # using opts.pkgdirs as a list, I think. - # FIXME - this needs to read the old repomd.xml to figure out where - # the files WERE to pass in the right fns. if self.conf.update: #build the paths opts = { @@ -288,10 +289,19 @@ class MetaDataGenerator: } if self.conf.skip_stat: opts['do_stat'] = False - + #and scan the old repo self.oldData = readMetadata.MetadataIndex(self.conf.outputdir, opts) - + + def _setup_grabber(self): + if not hasattr(self, '_grabber'): + self._grabber = grabber.URLGrabber() + + return self._grabber + + grabber = property(fget = lambda self: self._setup_grabber()) + + def doPkgMetadata(self): """all the heavy lifting for the package metadata""" if self.conf.update: @@ -342,18 +352,38 @@ class MetaDataGenerator: return fo - def read_in_package(self, rpmfile, pkgpath=None): + def read_in_package(self, rpmfile, pkgpath=None, reldir=None): """rpmfile == relative path to file from self.packge_dir""" - # TODO/FIXME - # consider adding a routine to download the package from a remote location - # to a tempdir, operate on it, then use that location as a the baseurl - # for the package. That would make it possible to have repos entirely - # comprised of remote packages. + remote_package = False + baseurl = self.conf.baseurl if not pkgpath: pkgpath = self.package_dir - rpmfile = '%s/%s' % (pkgpath, rpmfile) + if not rpmfile.strip(): + raise MDError, "Blank filename passed in, skipping" + + if rpmfile.find("://") != -1: + remote_package = True + + if not hasattr(self, 'tempdir'): + self.tempdir = tempfile.mkdtemp() + + pkgname = os.path.basename(rpmfile) + baseurl = os.path.dirname(rpmfile) + reldir = self.tempdir + dest = os.path.join(self.tempdir, pkgname) + if not self.conf.quiet: + self.callback.log('\nDownloading %s' % rpmfile) + try: + rpmfile = self.grabber.urlgrab(rpmfile, dest) + except grabber.URLGrabError, e: + raise MDError, "Unable to retrieve remote package %s: %s" %(rpmfile, e) + + + else: + rpmfile = '%s/%s' % (pkgpath, rpmfile) + try: po = yumbased.CreateRepoPackage(self.ts, rpmfile) except Errors.MiscError, e: @@ -362,11 +392,8 @@ class MetaDataGenerator: # you can do it po.crp_changelog_limit = self.conf.changelog_limit po.crp_cachedir = self.conf.cachedir - - # FIXME if we wanted to put in a baseurl-per-package here is where - # we should do it - # it would be easy to have a lookup dict in the MetaDataConfig object - # and work down from there for the baseurl + po.crp_baseurl = baseurl + po.crp_reldir = reldir if po.checksum in (None, ""): raise MDError, "No Package ID found for package %s, not going to add it" % e @@ -377,6 +404,7 @@ class MetaDataGenerator: if not pkglist: pkglist = self.conf.pkglist + if not pkgpath: directory=self.conf.directory else: @@ -385,13 +413,16 @@ class MetaDataGenerator: for pkg in pkglist: current+=1 recycled = False - + # look to see if we can get the data from the old repodata # if so write this one out that way if self.conf.update: #see if we can pull the nodes from the old repo #print self.oldData.basenodes.keys() - nodes = self.oldData.getNodes(pkg) + old_pkg = pkg + if pkg.find("://") != -1: + old_pkg = os.path.basename(pkg) + nodes = self.oldData.getNodes(old_pkg) if nodes is not None: recycled = True @@ -399,17 +430,19 @@ class MetaDataGenerator: # otherwise do it individually if not recycled: #scan rpm files + if not pkgpath: + reldir = os.path.join(self.conf.basedir, directory) + else: + reldir = pkgpath + try: - po = self.read_in_package(pkg, pkgpath=pkgpath) + po = self.read_in_package(pkg, pkgpath=pkgpath, reldir=reldir) except MDError, e: # need to say something here self.callback.errorlog("\nError %s: %s\n" % (pkg, e)) continue - if not pkgpath: - reldir = os.path.join(self.conf.basedir, directory) - else: - reldir = pkgpath - self.primaryfile.write(po.do_primary_xml_dump(reldir, baseurl=self.conf.baseurl)) + + self.primaryfile.write(po.do_primary_xml_dump()) self.flfile.write(po.do_filelists_xml_dump()) self.otherfile.write(po.do_other_xml_dump()) else: diff --git a/createrepo/yumbased.py b/createrepo/yumbased.py index 8f424fd..17af4e6 100644 --- a/createrepo/yumbased.py +++ b/createrepo/yumbased.py @@ -167,15 +167,11 @@ class CreateRepoPackage(YumLocalPackage): hdrend = property(fget=lambda self: self._get_header_byte_range()[1]) hdrstart = property(fget=lambda self: self._get_header_byte_range()[0]) - def _dump_base_items(self, basedir, baseurl=None): - """Takes an optional baseurl and required basedir. - basedir is the relative path to remove from the location - baseurl is whether or not this package already has a - baseurl rather than just '.'""" + def _dump_base_items(self): # if we start seeing fullpaths in the location tag - this is the culprit - if self.localpath.startswith(basedir): - relpath = self.localpath.replace(basedir, '') + if self.crp_reldir and self.localpath.startswith(self.crp_reldir): + relpath = self.localpath.replace(self.crp_reldir, '') if relpath[0] == '/': relpath = relpath[1:] else: relpath = self.localpath @@ -204,8 +200,8 @@ class CreateRepoPackage(YumLocalPackage): self.archivesize) - if baseurl: - msg += """\n""" % (self._xml(baseurl), relpath) + if self.crp_baseurl: + msg += """\n""" % (self._xml(self.crp_baseurl), relpath) else: msg += """\n""" % relpath @@ -379,9 +375,9 @@ class CreateRepoPackage(YumLocalPackage): del c return msg - def do_primary_xml_dump(self, basedir, baseurl=None): + def do_primary_xml_dump(self): msg = """\n""" - msg += self._dump_base_items(basedir, baseurl) + msg += self._dump_base_items() msg += self._dump_format_items() msg += """\n""" return msg diff --git a/genpkgmetadata.py b/genpkgmetadata.py index ed369da..e9de6b3 100755 --- a/genpkgmetadata.py +++ b/genpkgmetadata.py @@ -20,6 +20,7 @@ import os import sys +import re from optparse import OptionParser import createrepo @@ -118,7 +119,9 @@ def parseArgs(args, conf): if conf.pkglist: pfo = open(conf.pkglist, 'r') for line in pfo.readlines(): - line = line.replace('\n', '') + line = line.strip() + if re.match('^\s*\#.*', line) or re.match('^\s*$', line): + continue lst.append(line) pfo.close()