From: biao716.wang Date: Sat, 20 Nov 2021 12:40:44 +0000 (+0900) Subject: rollback to 0.9.9 version X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=1adb094a84bb56b73c8d11aaad7c302c15bb58aa;p=tools%2Fcreaterepo.git rollback to 0.9.9 version Change-Id: I8a5d2a199823a8f6f906bfff6f1857d97cfe0c3d Signed-off-by: biao716.wang --- diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..d18ef02 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,4 @@ +Seth Vidal +Luke Macken +James Antill +Paul Nasrat diff --git a/ChangeLog b/ChangeLog index 2321449..fb537ec 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,310 @@ +2011-01-26 Seth Vidal + + * createrepo.spec, createrepo/__init__.py: mark as 0.9.9 + +2011-01-26 Seth Vidal + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Override timestamp check on repos. for mergerepo (like repodiff). + Add createrepo --workers (non)completion. Add modifyrepo option + completion. + +2011-01-21 James Antill + + * createrepo/merge.py: Override timestamp check on repos. for + mergerepo (like repodiff). + +2011-01-03 Seth Vidal + + * createrepo/__init__.py: make sure when we want to look for rpms we + say .rpm not rpm b/c with the latter we catch .drpm files, too. :( + +2010-11-02 Ville Skyttä + + * createrepo.bash: Add createrepo --workers (non)completion. + +2010-11-02 Ville Skyttä + + * createrepo.bash: Add modifyrepo option completion. + +2010-10-08 Seth Vidal + + * createrepo.spec, createrepo/__init__.py: - add yum 3.2.29 requirement b/c of the small change I needed to + repoMDObject.py - set it to use /usr/share/createrepo/worker.py + +2010-10-07 Seth Vidal + + * createrepo/__init__.py: remove libxml2 import from __init__.py :) + + +2010-10-07 Seth Vidal + + * createrepo/__init__.py: make createrepo use the repomd/repodata + mechanism from yum for making a repomd.xml which simplifies the code + dramatically since we don't have to mess with xml in here. + +2010-10-07 Seth Vidal + + * modifyrepo.py: fix up the usage output for modifyrepo + +2010-09-10 Seth Vidal + + * createrepo/__init__.py, worker.py: - make sure we handle remote_url pkgs correctly until we get the + worker hooked up to handle them - if there are no pkgs to handle, + don't launch workers with nothing to do. - give better output from + the workers and have them obey -v/-q - everyone loves callbacks! + +2010-09-09 Seth Vidal + + * Makefile, createrepo/__init__.py, createrepo/utils.py, + createrepo/yumbased.py, genpkgmetadata.py, worker.py: create a + worker script for createrepo so createrepo can fork off N processes + to handle the md gathering from pkgs. This should speed up results + on systems which have been cpubound on the createrepo process. If + you're io bound it won't help you at all, and MAY make it worse. + many misc issues to iron out here - not the least of which is the + callback output and gathering stdout/stderr from the workers + +2010-08-20 Seth Vidal + + * createrepo/__init__.py: handle broken locking on nfs target dirs + better if database is true. - sqlite dbs don't like being made on + locations without locking available. - if we know we're going to be + creating dbs then we should attempt to lock before doing anything + else and bail out nicely if we can't get an exclusive lock + +2010-08-19 Seth Vidal + + * createrepo.spec: require yum 3.2.28 due to the imports in + modifyrepo + +2010-08-19 Seth Vidal + + * docs/modifyrepo.1: document --mdtype option + +2010-08-19 Seth Vidal + + * modifyrepo.py: - add option parsing for --mdtype - use the yum repomd objects to + read/write the repomd.xml - add mdtype option to RepoMetadata.add() + method + +2010-06-11 Seth Vidal + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Don't use /usr/bin/env ... it's evil --database is now the default + for mergerepo too, have --no-database in completions instead. + +2010-06-11 Seth Vidal + + * createrepo/__init__.py: - add option to createrepo config to collapse libc.so.6 requires - + this will only work with yum 3.2.28 and beyond + +2010-06-03 James Antill + + * modifyrepo.py: Don't use /usr/bin/env ... it's evil + +2010-06-02 Ville Skyttä + + * createrepo.bash: --database is now the default for mergerepo too, + have --no-database in completions instead. + +2010-06-01 Seth Vidal + + * mergerepo.py: whoops - no-database shouldn't default to true! + +2010-06-01 Seth Vidal + + * mergerepo.py: add --no-database to mergrepo, too + +2010-05-31 Ville Skyttä + + * createrepo.bash: --database is now the default, have --no-database + in completions instead. + +2010-05-28 Seth Vidal + + * docs/createrepo.8: update the docs for --no-database + +2010-05-28 Seth Vidal + + * createrepo/__init__.py, genpkgmetadata.py: make -d/--database the + default add --no-database in case someone somewhere needs to do + that + +2010-04-26 Seth Vidal + + * createrepo/__init__.py: fixme comments about try/excepting the + database creation calls due to a weird issue with locks not working + on a nfs mount and createreepo tracing back with a TypeError of all + things + +2010-04-21 Seth Vidal + + * createrepo/__init__.py, createrepo/readMetadata.py: if we cannot + find one of the old repodata files make the warning look more dire + make sure we look for the 'repodata' subdir inside update_md_path + +2010-04-21 Seth Vidal + + * createrepo/__init__.py: when the update_md_path doesn't exist - + emit a warning of some kind - rather than a somewhat quieter message + from MetadataIndex() this is mostly to help jesse b/c he asked + nicely + +2010-04-16 Colin Walters + + * genpkgmetadata.py: if we're not a tty, don't use the progress + output + +2010-04-15 Seth Vidal + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Tell git to ignore tarballs. Document --repo in man page. Add + bash completion. + +2010-04-15 Seth Vidal + + * createrepo/__init__.py: - catch errors when moving the olddir out/back - if we get a + yumLocalPackage object in our pkglist we should record we read it. + +2010-03-05 Ville Skyttä + + * .gitignore: Tell git to ignore tarballs. + +2010-03-05 Ville Skyttä + + * docs/createrepo.8: Document --repo in man page. + +2010-02-19 Ville Skyttä + + * Makefile, createrepo.bash, createrepo.spec: Add bash completion. + +2010-03-05 Seth Vidal + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Trim trailing whitespace. + +2010-03-05 Seth Vidal + + * createrepo/__init__.py, genpkgmetadata.py: add repo tags and + --repo option to describe the repo itself. request from suse. + +2010-02-12 Ville Skyttä + + * createrepo/__init__.py, createrepo/deltarpms.py, + createrepo/merge.py, createrepo/readMetadata.py, + createrepo/utils.py, createrepo/yumbased.py, dmd.py, + genpkgmetadata.py, mergerepo.py, modifyrepo.py: Trim trailing + whitespace. + +2010-02-11 Seth Vidal + + * genpkgmetadata.py: add option for setting max-delta-rpm-size for + pkgs which are too large to be delta'd. + +2010-02-10 Seth Vidal + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Make *Emacs unsuspicious about trailing whitespace. Fix --exclude + -> --excludes typo. Add missing spaces in various help strings. + +2010-02-10 Seth Vidal + + * createrepo/__init__.py, genpkgmetadata.py: add --read-pkgs-list + option to output list of pkgs actually read. completely optional + and only really useful with --update or a --cachedir for what pkgs + DID get read/parsed. + +2010-02-09 Ville Skyttä + + * Makefile: Make *Emacs unsuspicious about trailing whitespace. + +2010-02-09 Ville Skyttä + + * docs/createrepo.8: Fix --exclude -> --excludes typo. + +2010-02-09 Ville Skyttä + + * genpkgmetadata.py: Add missing spaces in various help strings. + +2010-02-08 Seth Vidal + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Add missing space in --checkts help string. Ignore *.py[co]. + Remove outdated comment about --baseurl. + +2010-02-08 Seth Vidal + + * createrepo/__init__.py, docs/createrepo.8, genpkgmetadata.py: - make --unique-md-filenames the default - add simple-md-filenames + to be able to disable the above if desired + +2010-01-18 Ville Skyttä + + * genpkgmetadata.py: Add missing space in --checkts help string. + +2009-09-25 Ville Skyttä + + * .gitignore: Ignore *.py[co]. + +2009-01-26 Ville Skyttä + + * docs/createrepo.8: Remove outdated comment about --baseurl. At + least yum uses it nowadays. + +2010-01-07 Dennis Gregorovic + + * createrepo/readMetadata.py: Fixed, convert stat mtime to int so + comparison can work, --update, BZ 553030 + +2010-01-07 Dennis Gregorovic + + * createrepo/readMetadata.py: Convert stat mtime to int so + comparison can work, --update, BZ 553030 + +2010-01-06 Dennis Gregorovic + + * createrepo/__init__.py: Change baseurl of "old" packages on + update, when baseurl specified + +2009-10-05 Seth Vidal + + * createrepo/__init__.py: apply fix for + https://bugzilla.redhat.com/show_bug.cgi?id=527259 make sure we're + not only testing the first pkg. Test all of them until we find one + that is newer. + +2009-09-14 Seth Vidal + + * createrepo.spec: add requirement on python-deltarpm for new patch + from Bill. + +2009-09-14 Bill Nottingham + + * createrepo/deltarpms.py: createrepo patch to use the new deltarpm + python bindings + +2009-08-28 Seth Vidal + + * ChangeLog: changelog merge + 2009-08-28 Seth Vidal * createrepo.spec, createrepo/__init__.py: mark as 0.9.8 diff --git a/Makefile b/Makefile index 3e69eba..60bb9db 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PKGNAME = createrepo VERSION=$(shell awk '/Version:/ { print $$2 }' ${PKGNAME}.spec) RELEASE=$(shell awk '/Release:/ { print $$2 }' ${PKGNAME}.spec) CVSTAG=createrepo-$(subst .,_,$(VERSION)-$(RELEASE)) -PYTHON=python2 +PYTHON=python SUBDIRS = $(PKGNAME) bin docs PYFILES = $(wildcard *.py) @@ -47,11 +47,12 @@ RM = rm -f MODULES = $(srcdir)/genpkgmetadata.py \ $(srcdir)/modifyrepo.py \ - $(srcdir)/mergerepo.py + $(srcdir)/mergerepo.py \ + $(srcdir)/worker.py .SUFFIXES: .py .pyc .py.pyc: - $(PYTHON) -c "import py_compile; py_compile.compile($*.py)" + python -c "import py_compile; py_compile.compile($*.py)" all: $(MODULES) diff --git a/createrepo.bash b/createrepo.bash index 2a95e93..54ac8b2 100644 --- a/createrepo.bash +++ b/createrepo.bash @@ -5,7 +5,7 @@ _cr_createrepo() COMPREPLY=() case $3 in - --version|-h|--help|-u|--baseurl|--distro|--content|--repo|\ + --version|-h|--help|-u|--baseurl|--distro|--content|--repo|--workers|\ --revision|-x|--excludes|--changelog-limit|--max-delta-rpm-size) return 0 ;; @@ -39,12 +39,12 @@ _cr_createrepo() if [[ $2 == -* ]] ; then COMPREPLY=( $( compgen -W '--version --help --quiet --verbose --profile --excludes --basedir --baseurl --groupfile --checksum --pretty - --cachedir --checkts --database --update --update-md-path + --cachedir --checkts --no-database --update --update-md-path --skip-stat --split --pkglist --includepkg --outputdir --skip-symlinks --changelog-limit --unique-md-filenames --simple-md-filenames --distro --content --repo --revision --deltas --oldpackagedirs --num-deltas --read-pkgs-list - --max-delta-rpm-size' -- "$2" ) ) + --max-delta-rpm-size --workers' -- "$2" ) ) else COMPREPLY=( $( compgen -d -- "$2" ) ) fi @@ -65,7 +65,7 @@ _cr_mergerepo() ;; esac - COMPREPLY=( $( compgen -W '--version --help --repo --archlist --database + COMPREPLY=( $( compgen -W '--version --help --repo --archlist --no-database --outputdir --nogroups --noupdateinfo' -- "$2" ) ) } && complete -F _cr_mergerepo -o filenames mergerepo mergerepo.py @@ -74,7 +74,26 @@ _cr_modifyrepo() { COMPREPLY=() - case $COMP_CWORD in + case $3 in + --version|-h|--help|--mdtype) + return 0 + ;; + esac + + if [[ $2 == -* ]] ; then + COMPREPLY=( $( compgen -W '--version --help --mdtype' -- "$2" ) ) + return 0 + fi + + local i argnum=1 + for (( i=1; i < ${#COMP_WORDS[@]}-1; i++ )) ; do + if [[ ${COMP_WORDS[i]} != -* && + ${COMP_WORDS[i-1]} != @(=|--mdtype) ]]; then + argnum=$(( argnum+1 )) + fi + done + + case $argnum in 1) COMPREPLY=( $( compgen -f -o plusdirs -- "$2" ) ) return 0 diff --git a/createrepo.spec b/createrepo.spec new file mode 100644 index 0000000..1e491cd --- /dev/null +++ b/createrepo.spec @@ -0,0 +1,80 @@ +%{!?python_sitelib: %define python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} + +Summary: Creates a common metadata repository +Name: createrepo +Version: 0.9.9 +Release: 1 +License: GPL +Group: System Environment/Base +Source: %{name}-%{version}.tar.gz +URL: http://createrepo.baseurl.org/ +BuildRoot: %{_tmppath}/%{name}-%{version}root +BuildArchitectures: noarch +Requires: python >= 2.1, rpm-python, rpm >= 0:4.1.1, libxml2-python +Requires: yum-metadata-parser, yum >= 3.2.29, python-deltarpm + +%description +This utility will generate a common metadata repository from a directory of +rpm packages + +%prep +%setup -q + +%install +[ "$RPM_BUILD_ROOT" != "/" ] && rm -rf $RPM_BUILD_ROOT +make DESTDIR=$RPM_BUILD_ROOT sysconfdir=%{_sysconfdir} install + +%clean +[ "$RPM_BUILD_ROOT" != "/" ] && rm -rf $RPM_BUILD_ROOT + + +%files +%defattr(-, root, root) +%dir %{_datadir}/%{name} +%doc ChangeLog README COPYING COPYING.lib +%{_sysconfdir}/bash_completion.d/ +%{_datadir}/%{name}/* +%{_bindir}/%{name} +%{_bindir}/modifyrepo +%{_bindir}/mergerepo +%{_mandir}/man8/createrepo.8* +%{_mandir}/man1/modifyrepo.1* +%{_mandir}/man1/mergerepo.1* +%{python_sitelib}/createrepo + +%changelog +* Wed Jan 26 2011 Seth Vidal +- bump to 0.9.9 +- add worker.py + +* Thu Aug 19 2010 Seth Vidal +- increase yum requirement for the modifyrepo use of RepoMD, RepoData and RepoMDError + +* Fri Aug 28 2009 Seth Vidal +- 0.9.8 + +* Tue Mar 24 2009 Seth Vidal +- 0.9.7 + +* Fri Oct 17 2008 Seth Vidal +- add mergerepo - 0.9.6 + +* Mon Feb 18 2008 Seth Vidal +- 0.9.5 + +* Mon Jan 28 2008 Seth Vidal +- 0.9.4 + +* Tue Jan 22 2008 Seth Vidal +- 0.9.3 + +* Thu Jan 17 2008 Seth Vidal +- significant api changes + +* Tue Jan 8 2008 Seth Vidal +- 0.9.1 - lots of fixes +- cleanup changelog, too + +* Thu Dec 20 2007 Seth Vidal +- beginning of the new version + diff --git a/createrepo/Makefile b/createrepo/Makefile index ce948ae..d3d3a34 100644 --- a/createrepo/Makefile +++ b/createrepo/Makefile @@ -1,4 +1,4 @@ -PYTHON=python2 +PYTHON=python PACKAGE = $(shell basename `pwd`) PYFILES = $(wildcard *.py) PYVER := $(shell $(PYTHON) -c 'import sys; print "%.3s" %(sys.version)') diff --git a/createrepo/__init__.py b/createrepo/__init__.py index 33dcb99..8f2538e 100644 --- a/createrepo/__init__.py +++ b/createrepo/__init__.py @@ -16,7 +16,6 @@ import os import sys -import libxml2 import fnmatch import time import yumbased @@ -25,11 +24,14 @@ from bz2 import BZ2File from urlgrabber import grabber import tempfile import stat +import fcntl +import subprocess from yum import misc, Errors, to_unicode +from yum.repoMDObject import RepoMD, RepoMDError, RepoData from yum.sqlutils import executeSQL from yum.packageSack import MetaSack -from yum.packages import YumAvailablePackage +from yum.packages import YumAvailablePackage, YumLocalPackage import rpmUtils.transaction from utils import _, errorprint, MDError @@ -45,10 +47,10 @@ except ImportError: pass from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, \ - checksum_and_rename + checksum_and_rename, split_list_into_equal_chunks import deltarpms -__version__ = '0.10.9' +__version__ = '0.9.9' class MetaDataConfig(object): @@ -77,7 +79,7 @@ class MetaDataConfig(object): self.max_delta_rpm_size = 100000000 self.update_md_path = None self.skip_stat = False - self.database = False + self.database = True self.outputdir = None self.file_patterns = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$'] self.dir_patterns = ['.*bin\/.*', '^\/etc\/.*'] @@ -103,7 +105,12 @@ class MetaDataConfig(object): self.repo_tags = []# strings, forwhatever they are worth self.read_pkgs_list = None # filepath/name to write out list of pkgs # read in this run of createrepo - + self.collapse_glibc_requires = True + self.workers = 1 # number of workers to fork off to grab metadata from the pkgs + self.worker_cmd = '/usr/share/createrepo/worker.py' + + #self.worker_cmd = './worker.py' # helpful when testing + class SimpleMDCallBack(object): def errorlog(self, thing): print >> sys.stderr, thing @@ -136,7 +143,7 @@ class MetaDataGenerator: self.read_pkgs = [] if not self.conf.directory and not self.conf.directories: - raise MDError( "No directory given on which to run.") + raise MDError, "No directory given on which to run." if not self.conf.directories: # just makes things easier later self.conf.directories = [self.conf.directory] @@ -180,31 +187,43 @@ class MetaDataGenerator: testdir = os.path.join(self.conf.basedir, mydir) if not os.path.exists(testdir): - raise MDError( _('Directory %s must exist') % mydir) + raise MDError, _('Directory %s must exist') % mydir if not os.path.isdir(testdir): - raise MDError( _('%s must be a directory') % mydir) + raise MDError, _('%s must be a directory') % mydir if not os.access(self.conf.outputdir, os.W_OK): - raise MDError( _('Directory %s must be writable.') % self.conf.outputdir) + raise MDError, _('Directory %s must be writable.') % self.conf.outputdir temp_output = os.path.join(self.conf.outputdir, self.conf.tempdir) if not checkAndMakeDir(temp_output): - raise MDError( _('Cannot create/verify %s') % temp_output) + raise MDError, _('Cannot create/verify %s') % temp_output temp_final = os.path.join(self.conf.outputdir, self.conf.finaldir) if not checkAndMakeDir(temp_final): - raise MDError( _('Cannot create/verify %s') % temp_final) + raise MDError, _('Cannot create/verify %s') % temp_final + if self.conf.database: + # do flock test on temp_final, temp_output + # if it fails raise MDError + for direc in [temp_final, temp_output]: + f = open(direc + '/locktest', 'w') + try: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) + except (OSError, IOError), e: + raise MDError, _("Could not create exclusive lock in %s and sqlite database generation enabled. Is this path on nfs? Is your lockd running?") % direc + else: + os.unlink(direc + '/locktest') + if self.conf.deltas: temp_delta = os.path.join(self.conf.outputdir, self.conf.delta_relative) if not checkAndMakeDir(temp_delta): - raise MDError( _('Cannot create/verify %s') % temp_delta) + raise MDError, _('Cannot create/verify %s') % temp_delta self.conf.deltadir = temp_delta if os.path.exists(os.path.join(self.conf.outputdir, self.conf.olddir)): - raise MDError( _('Old data directory exists, please remove: %s') % self.conf.olddir) + raise MDError, _('Old data directory exists, please remove: %s') % self.conf.olddir # make sure we can write to where we want to write to: # and pickup the mdtimestamps while we're at it @@ -217,7 +236,7 @@ class MetaDataGenerator: direc)) if os.path.exists(filepath): if not os.access(filepath, os.W_OK): - raise MDError( _('error in must be able to write to metadata dir:\n -> %s') % filepath) + raise MDError, _('error in must be able to write to metadata dir:\n -> %s') % filepath if self.conf.checkts: # checking for repodata/repomd.xml - not just the data dir @@ -235,7 +254,7 @@ class MetaDataGenerator: a = os.path.join(self.package_dir, self.conf.groupfile) if not os.path.exists(a): - raise MDError( _('Error: groupfile %s cannot be found.' % a)) + raise MDError, _('Error: groupfile %s cannot be found.' % a) self.conf.groupfile = a @@ -244,7 +263,7 @@ class MetaDataGenerator: if not os.path.isabs(a): a = os.path.join(self.conf.outputdir, a) if not checkAndMakeDir(a): - raise MDError( _('Error: cannot open/write to cache dir %s' % a)) + raise MDError, _('Error: cannot open/write to cache dir %s' % a) self.conf.cachedir = a @@ -337,7 +356,12 @@ class MetaDataGenerator: opts['do_stat'] = False if self.conf.update_md_path: - old_repo_path = os.path.normpath(self.conf.update_md_path) + norm_u_md_path = os.path.normpath(self.conf.update_md_path) + u_md_repodata_path = norm_u_md_path + '/repodata' + if not os.path.exists(u_md_repodata_path): + msg = _('Warning: could not open update_md_path: %s') % u_md_repodata_path + self.callback.errorlog(msg) + old_repo_path = os.path.normpath(norm_u_md_path) else: old_repo_path = self.conf.outputdir @@ -371,7 +395,7 @@ class MetaDataGenerator: self.writeMetadataDocs(packages) self.closeMetadataDocs() except (IOError, OSError), e: - raise MDError( _('Cannot access/write repodata files: %s') % e) + raise MDError, _('Cannot access/write repodata files: %s') % e def openMetadataDocs(self): @@ -391,7 +415,6 @@ class MetaDataGenerator: fo = _gzipOpen(primaryfilepath, 'w') fo.write('\n') fo.write('' % self.pkgcount) return fo @@ -435,7 +458,7 @@ class MetaDataGenerator: pkgpath = self.package_dir if not rpmfile.strip(): - raise MDError( "Blank filename passed in, skipping") + raise MDError, "Blank filename passed in, skipping" if rpmfile.find("://") != -1: @@ -451,30 +474,34 @@ class MetaDataGenerator: try: rpmfile = self.grabber.urlgrab(rpmfile, dest) except grabber.URLGrabError, e: - raise MDError ("Unable to retrieve remote package %s: %s" % ( - rpmfile, e)) + raise MDError, "Unable to retrieve remote package %s: %s" % ( + rpmfile, e) else: rpmfile = '%s/%s' % (pkgpath, rpmfile) + external_data = { '_cachedir': self.conf.cachedir, + '_baseurl': baseurl, + '_reldir': reldir, + '_packagenumber': self.current_pkg, + '_collapse_libc_requires':self.conf.collapse_glibc_requires, + } + try: po = yumbased.CreateRepoPackage(self.ts, rpmfile, - sumtype=self.conf.sumtype) + sumtype=self.conf.sumtype, + external_data = external_data) except Errors.MiscError, e: - raise MDError( "Unable to open package: %s" % e) - # external info we need - po._cachedir = self.conf.cachedir - po._baseurl = baseurl - po._reldir = reldir - po._packagenumber = self.current_pkg + raise MDError, "Unable to open package: %s" % e + for r in po.requires_print: if r.startswith('rpmlib('): self.rpmlib_reqs[r] = 1 if po.checksum in (None, ""): - raise MDError( "No Package ID found for package %s, not going to" \ - " add it" % po) + raise MDError, "No Package ID found for package %s, not going to" \ + " add it" % po return po @@ -488,100 +515,172 @@ class MetaDataGenerator: else: directory = pkgpath - for pkg in pkglist: - self.current_pkg += 1 - recycled = False + # for worker/forked model + # iterate the pkglist - see which ones are handled by --update and let them + # go on their merry way + + newpkgs = [] + if self.conf.update: + # if we're in --update mode then only act on the new/changed pkgs + for pkg in pkglist: + self.current_pkg += 1 - # look to see if we can get the data from the old repodata - # if so write this one out that way - if self.conf.update: #see if we can pull the nodes from the old repo #print self.oldData.basenodes.keys() old_pkg = pkg if pkg.find("://") != -1: old_pkg = os.path.basename(pkg) nodes = self.oldData.getNodes(old_pkg) - if nodes is not None: - recycled = True - - # FIXME also open up the delta file - - # otherwise do it individually - if not recycled: - #scan rpm files - if not pkgpath: - reldir = os.path.join(self.conf.basedir, directory) - else: - reldir = pkgpath - - if not isinstance(pkg, YumAvailablePackage): - - try: - po = self.read_in_package(pkg, pkgpath=pkgpath, - reldir=reldir) - except MDError, e: - # need to say something here - self.callback.errorlog("\nError %s: %s\n" % (pkg, e)) - continue - # we can use deltas: - if self.conf.deltas: - self._do_delta_rpm_package(po) - self.read_pkgs.append(pkg) + if nodes is not None: # we have a match in the old metadata + if self.conf.verbose: + self.callback.log(_("Using data from old metadata for %s") + % pkg) + (primarynode, filenode, othernode) = nodes + + for node, outfile in ((primarynode, self.primaryfile), + (filenode, self.flfile), + (othernode, self.otherfile)): + if node is None: + break + + if self.conf.baseurl: + anode = node.children + while anode is not None: + if anode.type != "element": + anode = anode.next + continue + if anode.name == "location": + anode.setProp('xml:base', self.conf.baseurl) + anode = anode.next + output = node.serialize('UTF-8', self.conf.pretty) + if output: + outfile.write(output) + else: + if self.conf.verbose: + self.callback.log(_("empty serialize on write to" \ + "%s in %s") % (outfile, pkg)) + outfile.write('\n') + + self.oldData.freeNodes(pkg) + #FIXME - if we're in update and we have deltas enabled + # check the presto data for this pkg and write its info back out + # to our deltafile + continue else: - po = pkg - self.read_pkgs.append(po.localpath) + newpkgs.append(pkg) + else: + newpkgs = pkglist - if self.conf.database_only: - pass # disabled right now for sanity reasons (mine) - #po.do_sqlite_dump(self.md_sqlite) + # setup our reldir + if not pkgpath: + reldir = os.path.join(self.conf.basedir, directory) + else: + reldir = pkgpath + + # filter out those pkgs which are not files - but are pkgobjects + pkgfiles = [] + for pkg in newpkgs: + po = None + if isinstance(pkg, YumAvailablePackage): + po = pkg + self.read_pkgs.append(po.localpath) + + # if we're dealing with remote pkgs - pitch it over to doing + # them one at a time, for now. + elif pkg.find('://') != -1: + po = self.read_in_package(pkgfile, pkgpath=pkgpath, reldir=reldir) + self.read_pkgs.append(pkg) + + if po: + self.primaryfile.write(po.xml_dump_primary_metadata()) + self.flfile.write(po.xml_dump_filelists_metadata()) + self.otherfile.write(po.xml_dump_other_metadata( + clog_limit=self.conf.changelog_limit)) + continue + + pkgfiles.append(pkg) + + + if pkgfiles: + # divide that list by the number of workers and fork off that many + # workers to tmpdirs + # waitfor the workers to finish and as each one comes in + # open the files they created and write them out to our metadata + # add up the total pkg counts and return that value + worker_tmp_path = tempfile.mkdtemp() + worker_chunks = utils.split_list_into_equal_chunks(pkgfiles, self.conf.workers) + worker_cmd_dict = {} + worker_jobs = {} + base_worker_cmdline = [self.conf.worker_cmd, + '--pkgoptions=_reldir=%s' % reldir, + '--pkgoptions=_collapse_libc_requires=%s' % self.conf.collapse_glibc_requires, + '--pkgoptions=_cachedir=%s' % self.conf.cachedir, + '--pkgoptions=_baseurl=%s' % self.conf.baseurl, + '--globalopts=clog_limit=%s' % self.conf.changelog_limit,] + + if self.conf.quiet: + base_worker_cmdline.append('--quiet') + + if self.conf.verbose: + base_worker_cmdline.append('--verbose') + + for worker_num in range(self.conf.workers): + # make the worker directory + workercmdline = [] + workercmdline.extend(base_worker_cmdline) + thisdir = worker_tmp_path + '/' + str(worker_num) + if checkAndMakeDir(thisdir): + workercmdline.append('--tmpmdpath=%s' % thisdir) else: - self.primaryfile.write(po.xml_dump_primary_metadata()) - self.flfile.write(po.xml_dump_filelists_metadata()) - self.otherfile.write(po.xml_dump_other_metadata( - clog_limit=self.conf.changelog_limit)) - else: - if self.conf.verbose: - self.callback.log(_("Using data from old metadata for %s") - % pkg) - (primarynode, filenode, othernode) = nodes - - for node, outfile in ((primarynode, self.primaryfile), - (filenode, self.flfile), - (othernode, self.otherfile)): - if node is None: - break - - if self.conf.baseurl: - anode = node.children - while anode is not None: - if anode.type != "element": - anode = anode.next - continue - if anode.name == "location": - anode.setProp('xml:base', self.conf.baseurl) - anode = anode.next - - output = node.serialize('UTF-8', self.conf.pretty) - if output: - outfile.write(output) - else: - if self.conf.verbose: - self.callback.log(_("empty serialize on write to" \ - "%s in %s") % (outfile, pkg)) - outfile.write('\n') - - self.oldData.freeNodes(pkg) - #FIXME - if we're in update and we have deltas enabled - # check the presto data for this pkg and write its info back out - # to our deltafile - + raise MDError, "Unable to create worker path: %s" % thisdir + workercmdline.extend(worker_chunks[worker_num]) + worker_cmd_dict[worker_num] = workercmdline + + + + for (num, cmdline) in worker_cmd_dict.items(): + if not self.conf.quiet: + self.callback.log("Spawning worker %s with %s pkgs" % (num, + len(worker_chunks[num]))) + job = subprocess.Popen(cmdline, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + worker_jobs[num] = job + + gimmebreak = 0 + while gimmebreak != len(worker_jobs.keys()): + gimmebreak = 0 + for (num,job) in worker_jobs.items(): + if job.poll() is not None: + gimmebreak+=1 + line = job.stdout.readline() + if line: + self.callback.log('Worker %s: %s' % (num, line.rstrip())) + line = job.stderr.readline() + if line: + self.callback.errorlog('Worker %s: %s' % (num, line.rstrip())) + + if not self.conf.quiet: - if self.conf.verbose: - self.callback.log('%d/%d - %s' % (self.current_pkg, - self.pkgcount, pkg)) - else: - self.callback.progress(pkg, self.current_pkg, self.pkgcount) + self.callback.log("Workers Finished") + # finished with workers + # go to their dirs and add the contents + if not self.conf.quiet: + self.callback.log("Gathering worker results") + for num in range(self.conf.workers): + for (fn, fo) in (('primary.xml', self.primaryfile), + ('filelists.xml', self.flfile), + ('other.xml', self.otherfile)): + fnpath = worker_tmp_path + '/' + str(num) + '/' + fn + if os.path.exists(fnpath): + fo.write(open(fnpath, 'r').read()) + + + for pkgfile in pkgfiles: + if self.conf.deltas: + po = self.read_in_package(pkgfile, pkgpath=pkgpath, reldir=reldir) + self._do_delta_rpm_package(po) + self.read_pkgs.append(pkgfile) return self.current_pkg @@ -682,12 +781,12 @@ class MetaDataGenerator: def _get_old_package_dict(self): if hasattr(self, '_old_package_dict'): - return self._old_package_dict #pylint: disable=access-member-before-definition + return self._old_package_dict self._old_package_dict = {} opl = [] for d in self.conf.oldpackage_paths: - for f in self.getFileList(d, 'rpm'): + for f in self.getFileList(d, '.rpm'): fp = d + '/' + f fpstat = os.stat(fp) if int(fpstat[stat.ST_SIZE]) > self.conf.max_delta_rpm_size: @@ -710,7 +809,7 @@ class MetaDataGenerator: # tag targets = {} results = [] - for drpm_fn in self.getFileList(self.conf.deltadir, 'drpm'): + for drpm_fn in self.getFileList(self.conf.deltadir, '.drpm'): drpm_rel_fn = os.path.normpath(self.conf.delta_relative + '/' + drpm_fn) # this is annoying drpm_po = yumbased.CreateRepoPackage(self.ts, @@ -733,13 +832,11 @@ class MetaDataGenerator: return ' '.join(results) - def addArbitraryMetadata(self, mdfile, mdtype, xml_node, compress=True, - compress_type='gzip', attribs={}): - """add random metadata to the repodata dir and repomd.xml + def _createRepoDataObject(self, mdfile, mdtype, compress=True, + compress_type='gzip', attribs={}): + """return random metadata as RepoData object to be added to RepoMD mdfile = complete path to file mdtype = the metadata type to use - xml_node = the node of the repomd xml object to append this - data onto compress = compress the file before including it """ # copy the file over here @@ -775,75 +872,57 @@ class MetaDataGenerator: else: csum = open_csum - timest = os.stat(outfn)[8] - - # add all this garbage into the xml node like: - data = xml_node.newChild(None, 'data', None) - data.newProp('type', mdtype) - location = data.newChild(None, 'location', None) - if self.conf.baseurl is not None: - location.newProp('xml:base', self.conf.baseurl) - location.newProp('href', os.path.join(self.conf.finaldir, sfile)) - checksum = data.newChild(None, 'checksum', csum) - checksum.newProp('type', self.conf.sumtype) + thisdata = RepoData() + thisdata.type = mdtype + baseloc = None + thisdata.location = (self.conf.baseurl, os.path.join(self.conf.finaldir, sfile)) + thisdata.checksum = (self.conf.sumtype, csum) if compress: - opencsum = data.newChild(None, 'open-checksum', open_csum) - opencsum.newProp('type', self.conf.sumtype) - - timestamp = data.newChild(None, 'timestamp', str(timest)) - - # add the random stuff + thisdata.openchecksum = (self.conf.sumtype, open_csum) + + thisdata.size = str(os.stat(outfn).st_size) + thisdata.timestamp = str(os.stat(outfn).st_mtime) for (k, v) in attribs.items(): - data.newChild(None, k, str(v)) - + setattr(thisdata, k, str(v)) + + return thisdata + def doRepoMetadata(self): """wrapper to generate the repomd.xml file that stores the info on the other files""" - repodoc = libxml2.newDoc("1.0") - reporoot = repodoc.newChild(None, "repomd", None) - repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None) - reporoot.setNs(repons) - rpmns = reporoot.newNs("http://linux.duke.edu/metadata/rpm", 'rpm') + + repomd = RepoMD('repoid') + repomd.revision = self.conf.revision + repopath = os.path.join(self.conf.outputdir, self.conf.tempdir) repofilepath = os.path.join(repopath, self.conf.repomdfile) - susens = reporoot.newNs("http://novell.com/package/metadata/suse/common", 'suse') - - revision = reporoot.newChild(None, 'revision', self.conf.revision) - if self.conf.content_tags or self.conf.distro_tags or self.conf.repo_tags: - tags = reporoot.newChild(None, 'tags', None) - for item in self.conf.content_tags: - c_tags = tags.newChild(None, 'content', item) - for item in self.conf.repo_tags: - c_tags = tags.newChild(None, 'repo', item) - for (cpeid, item) in self.conf.distro_tags: - d_tags = tags.newChild(None, 'distro', item) - if cpeid: - d_tags.newProp('cpeid', cpeid) + + if self.conf.content_tags: + repomd.tags['content'] = self.conf.content_tags + if self.conf.distro_tags: + repomd.tags['distro'] = self.conf.distro_tags + # NOTE - test out the cpeid silliness here + if self.conf.repo_tags: + repomd.tags['repo'] = self.conf.repo_tags + sumtype = self.conf.sumtype - if self.conf.database_only: - workfiles = [] - db_workfiles = [(self.md_sqlite.pri_sqlite_file, 'primary_db'), - (self.md_sqlite.file_sqlite_file, 'filelists_db'), - (self.md_sqlite.other_sqlite_file, 'other_db')] - dbversion = '10' - else: - workfiles = [(self.conf.otherfile, 'other',), - (self.conf.filelistsfile, 'filelists'), - (self.conf.primaryfile, 'primary')] - db_workfiles = [] - repoid = 'garbageid' + workfiles = [(self.conf.otherfile, 'other',), + (self.conf.filelistsfile, 'filelists'), + (self.conf.primaryfile, 'primary')] if self.conf.deltas: workfiles.append((self.conf.deltafile, 'prestodelta')) + if self.conf.database: if not self.conf.quiet: self.callback.log('Generating sqlite DBs') try: dbversion = str(sqlitecachec.DBVERSION) except AttributeError: dbversion = '9' - rp = sqlitecachec.RepodataParserSqlite(repopath, repoid, None) + #FIXME - in theory some sort of try/except here + rp = sqlitecachec.RepodataParserSqlite(repopath, repomd.repoid, None) for (rpm_file, ftype) in workfiles: complete_path = os.path.join(repopath, rpm_file) @@ -869,12 +948,16 @@ class MetaDataGenerator: time.ctime())) if ftype == 'primary': + #FIXME - in theory some sort of try/except here + # TypeError appears to be raised, sometimes :( rp.getPrimary(complete_path, csum) elif ftype == 'filelists': + #FIXME and here rp.getFilelists(complete_path, csum) elif ftype == 'other': + #FIXME and here rp.getOtherdata(complete_path, csum) if ftype in ['primary', 'filelists', 'other']: @@ -913,82 +996,60 @@ class MetaDataGenerator: # add this data as a section to the repomdxml db_data_type = '%s_db' % ftype - data = reporoot.newChild(None, 'data', None) - data.newProp('type', db_data_type) - location = data.newChild(None, 'location', None) - - if self.conf.baseurl is not None: - location.newProp('xml:base', self.conf.baseurl) - - location.newProp('href', os.path.join(self.conf.finaldir, - compressed_name)) - checksum = data.newChild(None, 'checksum', - db_compressed_sums[ftype]) - checksum.newProp('type', sumtype) - db_tstamp = data.newChild(None, 'timestamp', - str(db_stat.st_mtime)) - data.newChild(None, 'size', str(db_stat.st_size)) - data.newChild(None, 'open-size', str(un_stat.st_size)) - unchecksum = data.newChild(None, 'open-checksum', - db_csums[ftype]) - unchecksum.newProp('type', sumtype) - database_version = data.newChild(None, 'database_version', - dbversion) + data = RepoData() + data.type = db_data_type + data.location = (self.conf.baseurl, + os.path.join(self.conf.finaldir, compressed_name)) + data.checksum = (sumtype, db_compressed_sums[ftype]) + data.timestamp = str(db_stat.st_mtime) + data.size = str(db_stat.st_size) + data.opensize = str(un_stat.st_size) + data.openchecksum = (sumtype, db_csums[ftype]) + data.dbversion = dbversion if self.conf.verbose: self.callback.log("Ending %s db creation: %s" % (ftype, time.ctime())) + repomd.repoData[data.type] = data + + data = RepoData() + data.type = ftype + data.checksum = (sumtype, csum) + data.timestamp = str(timestamp) + data.size = str(os.stat(os.path.join(repopath, rpm_file)).st_size) + data.opensize = str(unsize) + data.openchecksum = (sumtype, uncsum) - - - data = reporoot.newChild(None, 'data', None) - data.newProp('type', ftype) - - checksum = data.newChild(None, 'checksum', csum) - checksum.newProp('type', sumtype) - timestamp = data.newChild(None, 'timestamp', str(timestamp)) - size = os.stat(os.path.join(repopath, rpm_file)) - data.newChild(None, 'size', str(size.st_size)) - data.newChild(None, 'open-size', str(unsize)) - unchecksum = data.newChild(None, 'open-checksum', uncsum) - unchecksum.newProp('type', sumtype) - location = data.newChild(None, 'location', None) - if self.conf.baseurl is not None: - location.newProp('xml:base', self.conf.baseurl) if self.conf.unique_md_filenames: res_file = '%s-%s.xml.gz' % (csum, ftype) orig_file = os.path.join(repopath, rpm_file) dest_file = os.path.join(repopath, res_file) os.rename(orig_file, dest_file) - else: res_file = rpm_file - rpm_file = res_file + href = os.path.join(self.conf.finaldir, rpm_file) - location.newProp('href', os.path.join(self.conf.finaldir, rpm_file)) - + data.location = (self.conf.baseurl, href) + repomd.repoData[data.type] = data if not self.conf.quiet and self.conf.database: self.callback.log('Sqlite DBs complete') - for (fn, ftype) in db_workfiles: - attribs = {'database_version':dbversion} - self.addArbitraryMetadata(fn, ftype, reporoot, compress=True, - compress_type='bzip2', attribs=attribs) - try: - os.unlink(fn) - except (IOError, OSError), e: - pass - if self.conf.groupfile is not None: - self.addArbitraryMetadata(self.conf.groupfile, 'group_gz', reporoot) - self.addArbitraryMetadata(self.conf.groupfile, 'group', reporoot, - compress=False) + mdcontent = self._createRepoDataObject(self.conf.groupfile, 'group_gz') + repomd.repoData[mdcontent.type] = mdcontent + + mdcontent = self._createRepoDataObject(self.conf.groupfile, 'group', + compress=False) + repomd.repoData[mdcontent.type] = mdcontent + if self.conf.additional_metadata: for md_type, mdfile in self.conf.additional_metadata.items(): - self.addArbitraryMetadata(mdfile, md_type, reporoot) + mdcontent = self._createRepoDataObject(md_file, md_type) + repomd.repoData[mdcontent.type] = mdcontent + # FIXME - disabled until we decide how best to use this #if self.rpmlib_reqs: @@ -999,14 +1060,16 @@ class MetaDataGenerator: # save it down try: - repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1) - except: + fo = open(repofilepath, 'w') + fo.write(repomd.dump_xml()) + fo.close() + except (IOError, OSError, TypeError), e: self.callback.errorlog( _('Error saving temp file for repomd.xml: %s') % repofilepath) - raise MDError( 'Could not save temp file: %s' % repofilepath) - - del repodoc - + self.callback.errorlog('Error was: %s') % str(e) + fo.close() + raise MDError, 'Could not save temp file: %s' % repofilepath + def doFinalMove(self): """move the just-created repodata from .repodata to repodata @@ -1020,8 +1083,8 @@ class MetaDataGenerator: try: os.rename(output_final_dir, output_old_dir) except: - raise MDError (_('Error moving final %s to old dir %s' % ( - output_final_dir, output_old_dir))) + raise MDError, _('Error moving final %s to old dir %s' % ( + output_final_dir, output_old_dir)) output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir) @@ -1030,7 +1093,7 @@ class MetaDataGenerator: except: # put the old stuff back os.rename(output_old_dir, output_final_dir) - raise MDError( _('Error moving final metadata into place')) + raise MDError, _('Error moving final metadata into place') for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', 'groupfile']: @@ -1044,10 +1107,15 @@ class MetaDataGenerator: try: os.remove(oldfile) except OSError, e: - raise MDError( _( - 'Could not remove old metadata file: %s: %s') % (oldfile, e)) + raise MDError, _( + 'Could not remove old metadata file: %s: %s') % (oldfile, e) # Move everything else back from olddir (eg. repoview files) + try: + old_contents = os.listdir(output_old_dir) + except (OSError, IOError), e: + old_contents = [] + for f in os.listdir(output_old_dir): oldfile = os.path.join(output_old_dir, f) finalfile = os.path.join(output_final_dir, f) @@ -1069,15 +1137,15 @@ class MetaDataGenerator: else: os.remove(oldfile) except OSError, e: - raise MDError (_( - 'Could not remove old metadata file: %s: %s') % (oldfile, e)) + raise MDError, _( + 'Could not remove old metadata file: %s: %s') % (oldfile, e) else: try: os.rename(oldfile, finalfile) except OSError, e: msg = _('Could not restore old non-metadata file: %s -> %s') % (oldfile, finalfile) msg += _('Error was %s') % e - raise MDError( msg) + raise MDError, msg try: os.rmdir(output_old_dir) @@ -1105,8 +1173,8 @@ class MetaDataGenerator: try: self.md_sqlite = MetaDataSqlite(destdir) except sqlite.OperationalError, e: - raise MDError( _('Cannot create sqlite databases: %s.\n'\ - 'Maybe you need to clean up a .repodata dir?') % e) + raise MDError, _('Cannot create sqlite databases: %s.\n'\ + 'Maybe you need to clean up a .repodata dir?') % e @@ -1180,7 +1248,7 @@ class SplitMetaDataGenerator(MetaDataGenerator): self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, 1) self.closeMetadataDocs() except (IOError, OSError), e: - raise MDError( _('Cannot access/write repodata files: %s') % e) + raise MDError, _('Cannot access/write repodata files: %s') % e diff --git a/createrepo/deltarpms.py b/createrepo/deltarpms.py index 28c6ece..3edcbb5 100644 --- a/createrepo/deltarpms.py +++ b/createrepo/deltarpms.py @@ -18,7 +18,7 @@ import os.path import commands from yum import misc -# import deltarpm +import deltarpm from utils import MDError class DeltaRPMPackage: @@ -35,7 +35,7 @@ class DeltaRPMPackage: self.mtime = stats[8] del stats except OSError, e: - raise MDError( "Error Stat'ing file %s%s" % (basedir, filename)) + raise MDError, "Error Stat'ing file %s%s" % (basedir, filename) self.csum_type = 'sha256' self.relativepath = filename self.po = po @@ -62,11 +62,10 @@ class DeltaRPMPackage: return length def _getDRPMInfo(self, filename): - # d = deltarpm.readDeltaRPM(filename) - # self.oldnevrstring = d['old_nevr'] - # self.oldnevr = self._stringToNEVR(d['old_nevr']) - # self.sequence = d['seq'] - return + d = deltarpm.readDeltaRPM(filename) + self.oldnevrstring = d['old_nevr'] + self.oldnevr = self._stringToNEVR(d['old_nevr']) + self.sequence = d['seq'] def _stringToVersion(self, strng): i = strng.find(':') diff --git a/createrepo/merge.py b/createrepo/merge.py index 78514a3..b3b2ea1 100644 --- a/createrepo/merge.py +++ b/createrepo/merge.py @@ -86,7 +86,9 @@ class RepoMergeBase: for r in self.repolist: count +=1 rid = 'repo%s' % count - n = self.yumbase.add_enable_repo(rid, baseurls=[r]) + n = self.yumbase.add_enable_repo(rid, baseurls=[r], + metadata_expire=0, + timestamp_check=False) n._merge_rank = count #setup our sacks diff --git a/createrepo/readMetadata.py b/createrepo/readMetadata.py index 4f13662..27d3690 100644 --- a/createrepo/readMetadata.py +++ b/createrepo/readMetadata.py @@ -59,7 +59,7 @@ class MetadataIndex(object): for fn in self.files.values(): if not os.path.exists(fn): #cannot scan - errorprint(_("Previous repo file missing: %s") % fn) + errorprint(_("Warning: Old repodata file missing: %s") % fn) return root = libxml2.parseFile(self.files['base']).getRootElement() self._scanPackageNodes(root, self._handleBase) diff --git a/createrepo/utils.py b/createrepo/utils.py index df3e93a..995c3b9 100644 --- a/createrepo/utils.py +++ b/createrepo/utils.py @@ -73,7 +73,7 @@ def returnFD(filename): try: fdno = os.open(filename, os.O_RDONLY) except OSError: - raise MDError("Error opening file") + raise MDError, "Error opening file" return fdno def checkAndMakeDir(directory): @@ -123,6 +123,15 @@ def encodefiletypelist(filetypelist): result += ftl[x] return result +def split_list_into_equal_chunks(seq, num_chunks): + avg = len(seq) / float(num_chunks) + out = [] + last = 0.0 + while last < len(seq): + out.append(seq[int(last):int(last + avg)]) + last += avg + + return out class MDError(Exception): diff --git a/createrepo/yumbased.py b/createrepo/yumbased.py index e94b3a5..ac06196 100644 --- a/createrepo/yumbased.py +++ b/createrepo/yumbased.py @@ -26,10 +26,15 @@ import utils import tempfile class CreateRepoPackage(YumLocalPackage): - def __init__(self, ts, package, sumtype=None): + def __init__(self, ts, package, sumtype=None, external_data={}): YumLocalPackage.__init__(self, ts, package) if sumtype: self.checksum_type = sumtype + + if external_data: + for (key, val) in external_data.items(): + setattr(self, key, val) + def _do_checksum(self): """return a checksum for a package: @@ -44,7 +49,7 @@ class CreateRepoPackage(YumLocalPackage): return self._checksum # not using the cachedir - if not self._cachedir: + if not hasattr(self, '_cachedir') or not self._cachedir: self._checksum = misc.checksum(self.checksum_type, self.localpath) self._checksums = [(self.checksum_type, self._checksum, 1)] return self._checksum @@ -55,8 +60,8 @@ class CreateRepoPackage(YumLocalPackage): t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG])) if type(self.hdr[rpm.RPMTAG_SIGPGP]) is not types.NoneType: t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP])) - if type(self.hdr[rpm.RPMTAG_SHA1HEADER]) is not types.NoneType: - t.append("".join(self.hdr[rpm.RPMTAG_SHA1HEADER])) + if type(self.hdr[rpm.RPMTAG_HDRID]) is not types.NoneType: + t.append("".join(self.hdr[rpm.RPMTAG_HDRID])) kcsum = misc.Checksums(checksums=[self.checksum_type]) kcsum.update("".join(t)) @@ -70,7 +75,7 @@ class CreateRepoPackage(YumLocalPackage): csumo = open(csumfile, 'r') checksum = csumo.readline() csumo.close() - os.path.os.utime(csumfile, None) + else: checksum = misc.checksum(self.checksum_type, self.localpath) diff --git a/debian/changelog b/debian/changelog index 502ab62..fa11637 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,15 +1,9 @@ -createrepo (0.10.9) unstable; urgency=low +createrepo (0.9.9) unstable; urgency=low * Make modifyrepo insert size and open-size tag -- Yan Meng Wed, 17 Jun 2020 09:28:48 +0100 -createrepo (0.10.8) unstable; urgency=low - - * update to 0.10.8 - - -- Biao Wang Wed, 20 Nov 2019 09:28:48 +0100 - createrepo (0.9.8) unstable; urgency=low * update to 0.9.8 diff --git a/dmd.py b/dmd.py index 0e0c3b8..684bac6 100755 --- a/dmd.py +++ b/dmd.py @@ -19,7 +19,8 @@ # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. import sys -from lxml.etree import parse, tostring, Element #pylint: disable=E0611 +from lxml.etree import parse, tostring, Element + class MdType(object): def __init__(self, namespace, rootelem): diff --git a/docs/createrepo.8 b/docs/createrepo.8 index de9112f..e3c4c3b 100644 --- a/docs/createrepo.8 +++ b/docs/createrepo.8 @@ -67,7 +67,11 @@ Output version. Show help menu. .IP "\fB\-d --database\fP" -Generate sqlite databases for use with yum. +Generate sqlite databases for use with yum. This is now the default. + +.IP "\fB\--no-database\fP" +Do not generate sqlite databases in the repository. + .IP "\fB\-S --skip-symlinks\fP" Ignore symlinks of packages .IP "\fB\-s --checksum\fP" diff --git a/docs/modifyrepo.1 b/docs/modifyrepo.1 index 6d16b15..cc031f5 100644 --- a/docs/modifyrepo.1 +++ b/docs/modifyrepo.1 @@ -4,7 +4,7 @@ modifyrepo \- Modify a repomd (xml-rpm-metadata) repository .SH "SYNOPSIS" -\fBmodifyrepo\fP +\fBmodifyrepo\fP [options] .PP .SH "DESCRIPTION" @@ -12,10 +12,10 @@ modifyrepo \- Modify a repomd (xml-rpm-metadata) repository .SH "EXAMPLES" .PP -$ \fBmodifyrepo\fP metadata.xml /repository/repodata +$ \fBmodifyrepo\fP --mdtype=newmd metadata.xml /repository/repodata .br Wrote: /repository/repodata/metadata.xml.gz - type = metadata + type = newmd location = repodata/metadata.xml.gz checksum = 1d7ee93db2964e7f85e07ec19b3204591da1050c timestamp = 1196716296.0 @@ -31,10 +31,11 @@ Wrote: /repository/repodata/repomd.xml .SH "AUTHORS" .nf Luke Macken +Seth Vidal .fi .PP .SH "BUGS" Any bugs which are found should be emailed to the mailing list: -rpm-metadata@linux.duke.edu +rpm-metadata@lists.baseurl.org .fi diff --git a/genpkgmetadata.py b/genpkgmetadata.py index 68ee572..8c98191 100755 --- a/genpkgmetadata.py +++ b/genpkgmetadata.py @@ -64,8 +64,10 @@ def parse_args(args, conf): parser.add_option("-C", "--checkts", default=False, action="store_true", help="check timestamps on files vs the metadata to see " \ "if we need to update") - parser.add_option("-d", "--database", default=False, action="store_true", - help="create sqlite database files") + parser.add_option("-d", "--database", default=True, action="store_true", + help="create sqlite database files: now default, see --no-database to disable") + parser.add_option("--no-database", default=False, dest="nodatabase", action="store_true", + help="do not create sqlite dbs of metadata") # temporarily disabled #parser.add_option("--database-only", default=False, action="store_true", # dest='database_only', @@ -106,8 +108,8 @@ def parse_args(args, conf): action="append", help="tags to describe the repository itself") parser.add_option("--revision", default=None, help="user-specified revision for this repository") - #parser.add_option("--deltas", default=False, action="store_true", - # help="create delta rpms and metadata") + parser.add_option("--deltas", default=False, action="store_true", + help="create delta rpms and metadata") parser.add_option("--oldpackagedirs", default=[], dest="oldpackage_paths", action="append", help="paths to look for older pkgs to delta against") parser.add_option("--num-deltas", default=1, dest='num_deltas', type='int', @@ -117,6 +119,11 @@ def parse_args(args, conf): parser.add_option("--max-delta-rpm-size", default=100000000, dest='max_delta_rpm_size', type='int', help="max size of an rpm that to run deltarpm against (in bytes)") + + parser.add_option("--workers", default=1, + dest='workers', type='int', + help="number of workers to spawn to read rpms") + (opts, argsleft) = parser.parse_args(args) if len(argsleft) > 1 and not opts.split: errorprint(_('Error: Only one directory allowed per run.')) @@ -145,7 +152,10 @@ def parse_args(args, conf): if opts.simple_md_filenames: opts.unique_md_filenames = False - + + if opts.nodatabase: + opts.database = False + # let's switch over to using the conf object - put all the opts into it for opt in parser.option_list: if opt.dest is None: # this is fairly silly @@ -190,6 +200,9 @@ def parse_args(args, conf): class MDCallBack(object): """cli callback object for createrepo""" + def __init__(self): + self.__show_progress = os.isatty(1) + def errorlog(self, thing): """error log output""" print >> sys.stderr, thing @@ -200,6 +213,9 @@ class MDCallBack(object): def progress(self, item, current, total): """progress bar""" + + if not self.__show_progress: + return beg = "%*d/%d - " % (len(str(total)), current, total) left = 80 - len(beg) sys.stdout.write("\r%s%-*.*s" % (beg, left, left, item)) diff --git a/mergerepo.py b/mergerepo.py index 80ab504..05e5f5e 100755 --- a/mergerepo.py +++ b/mergerepo.py @@ -39,7 +39,8 @@ def parse_args(args): help="repo url") parser.add_option("-a", "--archlist", default=[], action="append", help="Defaults to all arches - otherwise specify arches") - parser.add_option("-d", "--database", default=False, action="store_true") + parser.add_option("-d", "--database", default=True, action="store_true") + parser.add_option( "--no-database", default=False, action="store_true", dest="nodatabase") parser.add_option("-o", "--outputdir", default=None, help="Location to create the repository") parser.add_option("", "--nogroups", default=False, action="store_true", @@ -70,8 +71,8 @@ def main(args): rmbase.archlist = opts.archlist if opts.outputdir: rmbase.outputdir = opts.outputdir - if opts.database: - rmbase.mdconf.database = True + if opts.nodatabase: + rmbase.mdconf.database = False if opts.nogroups: rmbase.groups = False if opts.noupdateinfo: diff --git a/modifyrepo.py b/modifyrepo.py index 6771fe4..17094a4 100755 --- a/modifyrepo.py +++ b/modifyrepo.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # This tools is used to insert arbitrary metadata into an RPM repository. # Example: # ./modifyrepo.py updateinfo.xml myrepo/repodata @@ -23,11 +23,13 @@ import os import sys - +from createrepo import __version__ from createrepo.utils import checksum_and_rename, GzipFile, MDError from yum.misc import checksum +from yum.repoMDObject import RepoMD, RepoMDError, RepoData from xml.dom import minidom +from optparse import OptionParser class RepoMetadata: @@ -37,30 +39,25 @@ class RepoMetadata: self.repodir = os.path.abspath(repo) self.repomdxml = os.path.join(self.repodir, 'repomd.xml') self.checksum_type = 'sha256' + if not os.path.exists(self.repomdxml): - raise MDError('%s not found' % self.repomdxml) - self.doc = minidom.parse(self.repomdxml) - - def _insert_element(self, parent, name, attrs=None, text=None): - child = self.doc.createElement(name) - if not attrs: - attrs = {} - for item in attrs.items(): - child.setAttribute(item[0], item[1]) - if text: - txtnode = self.doc.createTextNode(text) - child.appendChild(txtnode) - parent.appendChild(child) - return child - - def add(self, metadata): + raise MDError, '%s not found' % self.repomdxml + + try: + self.repoobj = RepoMD(self.repodir) + self.repoobj.parse(self.repomdxml) + except RepoMDError, e: + raise MDError, 'Could not parse %s' % self.repomdxml + + + def add(self, metadata, mdtype=None): """ Insert arbitrary metadata into this repository. metadata can be either an xml.dom.minidom.Document object, or a filename. """ md = None if not metadata: - raise MDError( 'metadata cannot be None') + raise MDError, 'metadata cannot be None' if isinstance(metadata, minidom.Document): md = metadata.toxml() mdname = 'updateinfo.xml' @@ -74,14 +71,16 @@ class RepoMetadata: oldmd.close() mdname = os.path.basename(metadata) else: - raise MDError('%s not found' % metadata) + raise MDError, '%s not found' % metadata else: - raise MDError( 'invalid metadata type') + raise MDError, 'invalid metadata type' ## Compress the metadata and move it into the repodata if not mdname.endswith('.gz'): mdname += '.gz' - mdtype = mdname.split('.')[0] + if not mdtype: + mdtype = mdname.split('.')[0] + destmd = os.path.join(self.repodir, mdname) newmd = GzipFile(filename=destmd, mode='wb') newmd.write(md) @@ -89,73 +88,61 @@ class RepoMetadata: print "Wrote:", destmd open_csum = checksum(self.checksum_type, metadata) - - csum, destmd = checksum_and_rename(destmd, self.checksum_type) base_destmd = os.path.basename(destmd) ## Remove any stale metadata - for elem in self.doc.getElementsByTagName('data'): - if elem.attributes['type'].value == mdtype: - self.doc.firstChild.removeChild(elem) - - ## Build the metadata - root = self.doc.firstChild - root.appendChild(self.doc.createTextNode(" ")) - data = self._insert_element(root, 'data', attrs={ 'type' : mdtype }) - data.appendChild(self.doc.createTextNode("\n ")) - - self._insert_element(data, 'location', - attrs={ 'href' : 'repodata/' + base_destmd }) - data.appendChild(self.doc.createTextNode("\n ")) - self._insert_element(data, 'checksum', - attrs={ 'type' : self.checksum_type }, - text=csum) - data.appendChild(self.doc.createTextNode("\n ")) - self._insert_element(data, 'timestamp', - text=str(os.stat(destmd).st_mtime)) - data.appendChild(self.doc.createTextNode("\n ")) - self._insert_element(data, 'size', - text=str(os.stat(destmd).st_size)) - data.appendChild(self.doc.createTextNode("\n ")) - self._insert_element(data, 'open-size', - text=str(os.stat(metadata).st_size)) - data.appendChild(self.doc.createTextNode("\n ")) - self._insert_element(data, 'open-checksum', - attrs={ 'type' : self.checksum_type }, - text=open_csum) - - data.appendChild(self.doc.createTextNode("\n ")) - root.appendChild(self.doc.createTextNode("\n")) - - print " type =", mdtype - print " location =", 'repodata/' + mdname - print " checksum =", csum - print " timestamp =", str(os.stat(destmd).st_mtime) - print " size =", str(os.stat(destmd).st_size) - print " open-size =", str(os.stat(metadata).st_size) - print " open-checksum =", open_csum + if mdtype in self.repoobj.repoData: + del self.repoobj.repoData[mdtype] + + + new_rd = RepoData() + new_rd.type = mdtype + new_rd.location = (None, 'repodata/' + base_destmd) + new_rd.checksum = (self.checksum_type, csum) + new_rd.openchecksum = (self.checksum_type, open_csum) + new_rd.size = str(os.stat(destmd).st_size) + new_rd.timestamp = str(os.stat(destmd).st_mtime) + self.repoobj.repoData[new_rd.type] = new_rd + + print " type =", new_rd.type + print " location =", new_rd.location[1] + print " checksum =", new_rd.checksum[1] + print " timestamp =", new_rd.timestamp + print " open-checksum =", new_rd.openchecksum[1] ## Write the updated repomd.xml outmd = file(self.repomdxml, 'w') - self.doc.writexml(outmd) - outmd.write("\n") + outmd.write(self.repoobj.dump_xml()) outmd.close() print "Wrote:", self.repomdxml -if __name__ == '__main__': - if len(sys.argv) != 3 or '-h' in sys.argv: - print "Usage: %s " % sys.argv[0] - sys.exit() +def main(args): + parser = OptionParser(version='modifyrepo version %s' % __version__) + # query options + parser.add_option("--mdtype", dest='mdtype', + help="specific datatype of the metadata, will be derived from the filename if not specified") + parser.usage = "modifyrepo [options] " + + (opts, argsleft) = parser.parse_args(args) + if len(argsleft) != 2: + parser.print_usage() + return 0 + metadata = argsleft[0] + repodir = argsleft[1] try: - repomd = RepoMetadata(sys.argv[2]) + repomd = RepoMetadata(repodir) except MDError, e: print "Could not access repository: %s" % str(e) - sys.exit(1) + return 1 try: - repomd.add(sys.argv[1]) + repomd.add(metadata, mdtype=opts.mdtype) except MDError, e: - print "Could not add metadata from file %s: %s" % (sys.argv[1], str(e)) - sys.exit(1) + print "Could not add metadata from file %s: %s" % (metadata, str(e)) + return 1 + +if __name__ == '__main__': + ret = main(sys.argv[1:]) + sys.exit(ret) diff --git a/packaging/createrepo.spec b/packaging/createrepo.spec index 1251b76..c9c7b63 100644 --- a/packaging/createrepo.spec +++ b/packaging/createrepo.spec @@ -22,8 +22,8 @@ Name: createrepo BuildRequires: python -Version: 0.10.9 -Release: 10.9 +Version: 0.9.9 +Release: 9.9 License: GPLv2+ Summary: Creates a Common Metadata Repository Group: System/Packages diff --git a/worker.py b/worker.py new file mode 100755 index 0000000..eb35ef7 --- /dev/null +++ b/worker.py @@ -0,0 +1,99 @@ +#!/usr/bin/python -tt + +import sys +import yum +import createrepo +import os +import rpmUtils +from optparse import OptionParser + + +# pass in dir to make tempdirs in +# make tempdir for this worker +# create 3 files in that tempdir +# return how many pkgs +# return on stderr where things went to hell + +#TODO - take most of read_in_package from createrepo and duplicate it here +# so we can do downloads, etc. +# then replace callers of read_in_package with forked callers of this +# and reassemble at the end + +def main(args): + parser = OptionParser() + parser.add_option('--tmpmdpath', default=None, + help="path where the outputs should be dumped for this worker") + parser.add_option("--pkgoptions", default=[], action='append', + help="pkgoptions in the format of key=value") + parser.add_option("--quiet", default=False, action='store_true', + help="only output errors and a total") + parser.add_option("--verbose", default=False, action='store_true', + help="output errors and a total") + parser.add_option("--globalopts", default=[], action='append', + help="general options in the format of key=value") + + + opts, pkgs = parser.parse_args(args) + external_data = {'_packagenumber': 1} + globalopts = {} + if not opts.tmpmdpath: + print >> sys.stderr, "tmpmdpath required for destination files" + sys.exit(1) + + + for strs in opts.pkgoptions: + k,v = strs.split('=') + if v in ['True', 'true', 'yes', '1', 1]: + v = True + elif v in ['False', 'false', 'no', '0', 0]: + v = False + elif v in ['None', 'none', '']: + v = None + external_data[k] = v + + for strs in opts.globalopts: + k,v = strs.split('=') + if v in ['True', 'true', 'yes', '1', 1]: + v = True + elif v in ['False', 'false', 'no', '0', 0]: + v = False + elif v in ['None', 'none', '']: + v = None + globalopts[k] = v + + + reldir = external_data['_reldir'] + ts = rpmUtils.transaction.initReadOnlyTransaction() + pri = open(opts.tmpmdpath + '/primary.xml' , 'w') + fl = open(opts.tmpmdpath + '/filelists.xml' , 'w') + other = open(opts.tmpmdpath + '/other.xml' , 'w') + + + for pkgfile in pkgs: + pkgpath = reldir + '/' + pkgfile + if not os.path.exists(pkgpath): + print >> sys.stderr, "File not found: %s" % pkgpath + continue + + try: + if not opts.quiet and opts.verbose: + print "reading %s" % (pkgfile) + + pkg = createrepo.yumbased.CreateRepoPackage(ts, package=pkgpath, + external_data=external_data) + pri.write(pkg.xml_dump_primary_metadata()) + fl.write(pkg.xml_dump_filelists_metadata()) + other.write(pkg.xml_dump_other_metadata(clog_limit= + globalopts.get('clog_limit', None))) + except yum.Errors.YumBaseError, e: + print >> sys.stderr, "Error: %s" % e + continue + else: + external_data['_packagenumber']+=1 + + pri.close() + fl.close() + other.close() + +if __name__ == "__main__": + main(sys.argv[1:])