fixed memory use problem

updated spec for 0.2
fixed makefile dumbness
fixed problems with broken symlinks

diff --git a/Makefile b/Makefile
index 9ff193d571bfff5cff26b3b3356063500bf5f50c..5883c594b487de9aaf222563a232862f8ac65120 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 PACKAGE = createrepo
-VERSION = 0.1
+VERSION = 0.2
 SHELL = /bin/sh
 top_srcdir = .
 srcdir = .
@@ -49,7 +49,7 @@ MODULES = $(srcdir)/genpkgmetadata.py \
 
 all: $(MODULES)
        for subdir in $(SUBDIRS) ; do \
-         $(MAKE) -C $$subdir ; \
+         $(MAKE) -C $$subdir VERSION=$(VERSION) PACKAGE=$(PACKAGE); \
        done
 
 check: 
@@ -58,7 +58,7 @@ check:
 install: all installdirs
        $(INSTALL_MODULES) $(srcdir)/$(MODULES) $(DESTDIR)$(pkgdatadir)
        for subdir in $(SUBDIRS) ; do \
-         $(MAKE) -C $$subdir install ; \
+         $(MAKE) -C $$subdir install VERSION=$(VERSION) PACKAGE=$(PACKAGE); \
        done
 
 installdirs:
@@ -72,13 +72,13 @@ uninstall:
          $(RM) $(pkgdatadir)/$$module ; \
        done
        for subdir in $(SUBDIRS) ; do \
-         $(MAKE) -C $$subdir uninstall ; \
+         $(MAKE) -C $$subdir uninstall VERSION=$(VERSION) PACKAGE=$(PACKAGE); \
        done
 
 clean:
        $(RM)  *.pyc *.pyo
        for subdir in $(SUBDIRS) ; do \
-         $(MAKE) -C $$subdir clean ; \
+         $(MAKE) -C $$subdir clean VERSION=$(VERSION) PACKAGE=$(PACKAGE); \
        done
 
 distclean: clean
@@ -86,7 +86,7 @@ distclean: clean
        $(RM) core
        $(RM) *~
        for subdir in $(SUBDIRS) ; do \
-         $(MAKE) -C $$subdir distclean ; \
+         $(MAKE) -C $$subdir distclean VERSION=$(VERSION) PACKAGE=$(PACKAGE); \
        done
 
 mostlyclean:
@@ -136,7 +136,7 @@ dailyfiles:
        $(srcdir)/$(PACKAGE).spec
        $(top_srcdir)/.disttmp/$$distdir
        for subdir in $(SUBDIRS) ; do \
-         $(MAKE) -C $$subdir dailyfiles ; \
+         $(MAKE) -C $$subdir dailyfiles VERSION=$(VERSION) PACKAGE=$(PACKAGE); \
        done
 
 distfiles:
@@ -147,7 +147,7 @@ distfiles:
        $(srcdir)/$(PACKAGE).spec \
        $(top_srcdir)/.disttmp/$$distdir
        for subdir in $(SUBDIRS) ; do \
-         $(MAKE) -C $$subdir distfiles ; \
+         $(MAKE) -C $$subdir distfiles VERSION=$(VERSION) PACKAGE=$(PACKAGE); \
        done
 
 archive: dist

diff --git a/bin/Makefile b/bin/Makefile
index 278c4698877ce12f2223d1e72250ecdd9a88db39..2e2ac2cf1a8e09f12c487584f21ad5ce169819a6 100644 (file)
--- a/bin/Makefile
+++ b/bin/Makefile
@@ -1,5 +1,3 @@
-PACKAGE = createrepo
-VERSION = 0.1
 SHELL = /bin/sh
 top_srcdir = ..
 srcdir = ../bin
@@ -38,15 +36,15 @@ INSTALL_MODULES = $(INSTALL) -m 755 -D
 RM             = rm -f
 
 
-all: $(srcdir)/createrepo
+all: $(srcdir)/$(PACKAGE)
 
 
 install: all installdirs
-       $(INSTALL_BIN) $(srcdir)/createrepo $(DESTDIR)$(bindir)/createrepo
+       $(INSTALL_BIN) $(srcdir)/$(PACKAGE) $(DESTDIR)$(bindir)/$(PACKAGE)
 
 
 uninstall:
-       $(RM) $(bindir)/createrepo
+       $(RM) $(bindir)/$(PACKAGE)
 
 
 
@@ -71,7 +69,7 @@ distfiles:
        distdir=$(PACKAGE)-$(VERSION); \
        mkdir $(top_srcdir)/.disttmp/$$distdir/bin;\
        cp \
-       $(srcdir)/createrepo \
+       $(srcdir)/$(PACKAGE) \
        $(srcdir)/Makefile \
        $(top_srcdir)/.disttmp/$$distdir/bin
 
@@ -79,7 +77,7 @@ dailyfiles:
        distdir=$(PACKAGE); \
        mkdir $(top_srcdir)/.disttmp/$$distdir/bin;\
        cp \
-       $(srcdir)/createrepo \
+       $(srcdir)/$(PACKAGE) \
        $(srcdir)/Makefile \
        $(top_srcdir)/.disttmp/$$distdir/bin
 

diff --git a/createrepo.spec b/createrepo.spec
index a08db1222f9eca5b26d58c7c8b71f2d509c9495a..ee9178872f7a0ada4fc22bdfdccce48e84f17b91 100644 (file)
--- a/createrepo.spec
+++ b/createrepo.spec
@@ -1,6 +1,6 @@
 Summary: Creates a common metadata repository
 Name: createrepo
-Version: 0.1
+Version: 0.2
 Release: 1
 License: GPL
 Group: System Environment/Base
@@ -32,6 +32,9 @@ rpm packages
 %{_bindir}/%{name}
 
 %changelog
+* Tue Jan 13 2004 Seth Vidal <skvidal@phy.duke.edu>
+- 0.2 - 
+
 * Sat Jan 10 2004 Seth Vidal <skvidal@phy.duke.edu>
-- first packaginging
+- first packaging
 

diff --git a/dumpMetadata.py b/dumpMetadata.py
index eb53c55bc91738e4553db0ea380eaef7a107c76a..564b14fd44882179ab738060fe8fdeb19b8aa11c 100644 (file)
--- a/dumpMetadata.py
+++ b/dumpMetadata.py
@@ -541,7 +541,9 @@ def generateXML(doc, node, rpmObj, sumtype):
         directory = xmlCleanString(doc, directory)
         files.addContent(directory)
         files.newProp('type', 'dir')
-
+    
+    return pkgNode
+    
 def fileListXML(doc, node, rpmObj):
     pkg = node.newChild(None, 'package', None)
     pkg.newProp('pkgid', rpmObj.pkgid)
@@ -565,7 +567,8 @@ def fileListXML(doc, node, rpmObj):
         ghost = xmlCleanString(doc, ghost)
         files.addContent(ghost)
         files.newProp('type', 'ghost')
-              
+    return pkg
+       
 def otherXML(doc, node, rpmObj):
     pkg = node.newChild(None, 'package', None)
     pkg.newProp('pkgid', rpmObj.pkgid)
@@ -582,7 +585,8 @@ def otherXML(doc, node, rpmObj):
         clog.addContent(text)
         clog.newProp('author', utf8String(name))
         clog.newProp('date', str(time))
-
+    return pkg
+    
 def repoXML(node, cmds):
     """generate the repomd.xml file that stores the info on the other files"""
     sumtype = cmds['sumtype']

diff --git a/genpkgmetadata.py b/genpkgmetadata.py
index 3b406b31931f5d7e035a60b209fba0a1e739d89e..f4de6f2781062569948f550e12b0fbf9b8e30061 100755 (executable)
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -27,8 +27,12 @@ import rpm
 import libxml2
 import string
 import fnmatch
-import dumpMetadata
+# done to fix gzip randomly changing the checksum
+import gzip
+from zlib import error as zlibError
+from gzip import write32u, FNAME
 
+import dumpMetadata
 __version__ = '0.1'
 
 def errorprint(stuff):
@@ -96,6 +100,27 @@ def trimRpms(rpms, excludeGlobs):
     # print 'Post-Trim Len: %d' % len(rpms)
     return rpms
 
+# this is done to make the hdr writing _more_ sane for rsync users especially
+__all__ = ["GzipFile","open"]
+
+class GzipFile(gzip.GzipFile):
+    def _write_gzip_header(self):
+        self.fileobj.write('\037\213')             # magic header
+        self.fileobj.write('\010')                 # compression method
+        fname = self.filename[:-3]
+        flags = 0
+        if fname:
+            flags = FNAME
+        self.fileobj.write(chr(flags))
+        write32u(self.fileobj, long(0))
+        self.fileobj.write('\002')
+        self.fileobj.write('\377')
+        if fname:
+            self.fileobj.write(fname + '\000')
+
+
+def _gzipOpen(filename, mode="rb", compresslevel=9):
+    return GzipFile(filename, mode, compresslevel)
     
 def parseArgs(args):
     """
@@ -174,17 +199,29 @@ def doPkgMetadata(cmds, ts):
     baseroot =  basedoc.newChild(None, "metadata", None)
     basens = baseroot.newNs('http://linux.duke.edu/metadata/common', None)
     baseroot.setNs(basens)
+    basefile = _gzipOpen('.primary.xml.gz', 'w')
+    basefile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+    basefile.write('<metadata xmlns="http://linux.duke.edu/metadata/common">\n')
+
     # setup the file list doc
     filesdoc = libxml2.newDoc("1.0")
     filesroot = filesdoc.newChild(None, "filelists", None)
     filesns = filesroot.newNs('http://linux.duke.edu/metadata/filelists', None)
     filesroot.setNs(filesns)
+    flfile = _gzipOpen('.filelists.xml.gz', 'w')    
+    flfile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+    flfile.write('<filelists xmlns="http://linux.duke.edu/metadata/filelists">\n')
+    
+    
     # setup the other doc
     otherdoc = libxml2.newDoc("1.0")
     otherroot = otherdoc.newChild(None, "otherdata", None)
     otherns = otherroot.newNs('http://linux.duke.edu/metadata/other', None)
     otherroot.setNs(otherns)
-
+    otherfile = _gzipOpen('.other.xml.gz', 'w')
+    otherfile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+    otherfile.write('<otherdata xmlns="http://linux.duke.edu/metadata/other">\n')
+    
     files = []
     files = getFileList('./', '.rpm', files)
     files = trimRpms(files, cmds['excludes'])
@@ -206,38 +243,63 @@ def doPkgMetadata(cmds, ts):
             continue
         else:
             try:
-                dumpMetadata.generateXML(basedoc, baseroot, mdobj, cmds['sumtype'])
+                node = dumpMetadata.generateXML(basedoc, baseroot, mdobj, cmds['sumtype'])
             except dumpMetadata.MDError, e:
                 errorprint(_('\nAn error occurred creating primary metadata: %s') % e)
                 continue
+            else:
+                output = node.serialize(None, 1)
+                basefile.write(output)
+                node.unlinkNode()
+                node.freeNode()
+                del node
+
             try:
-                dumpMetadata.fileListXML(filesdoc, filesroot, mdobj)
+                node = dumpMetadata.fileListXML(filesdoc, filesroot, mdobj)
             except dumpMetadata.MDError, e:
                 errorprint(_('\nAn error occurred creating filelists: %s') % e)
                 continue
+            else:
+                output = node.serialize(None, 1)
+                flfile.write(output)
+                node.unlinkNode()
+                node.freeNode()
+                del node
+
             try:
-                dumpMetadata.otherXML(otherdoc, otherroot, mdobj)
+                node = dumpMetadata.otherXML(otherdoc, otherroot, mdobj)
             except dumpMetadata.MDError, e:
                 errorprint(_('\nAn error occurred: %s') % e)
                 continue
+            else:
+                output = node.serialize(None, 1)
+                otherfile.write(output)
+                node.unlinkNode()
+                node.freeNode()
+                del node
+
+        
     if not cmds['quiet']:
         print ''
         
     # save them up to the tmp locations:
-    basedoc.setDocCompressMode(9)                
     if not cmds['quiet']:
         print _('Saving Primary metadata')
-    basedoc.saveFormatFileEnc('.primary.xml.gz', 'UTF-8', 1)
+    basefile.write('\n</metadata>')
+    basefile.close()
+    basedoc.freeDoc()
     
-    filesdoc.setDocCompressMode(9)
     if not cmds['quiet']:
         print _('Saving file lists metadata')
-    filesdoc.saveFormatFileEnc('.filelists.xml.gz', 'UTF-8', 1)
+    flfile.write('\n</filelists>')
+    flfile.close()
+    filesdoc.freeDoc()
     
-    otherdoc.setDocCompressMode(9)
     if not cmds['quiet']:
         print _('Saving other metadata')
-    otherdoc.saveFormatFileEnc('.other.xml.gz', 'UTF-8', 1)
+    otherfile.write('\n</otherdata>')
+    otherfile.close()
+    otherdoc.freeDoc()
     
     # move them to their final locations
     for (tmp, dest) in [('.other.xml.gz', cmds['otherfile']), 
@@ -342,8 +404,11 @@ def main(args):
 
         
 if __name__ == "__main__":
-    if sys.argv[1] == 'profile':
-        import profile
-        profile.run('main(sys.argv[2:])')
-    else:    
+    if len(sys.argv) > 1:
+        if sys.argv[1] == 'profile':
+            import profile
+            profile.run('main(sys.argv[2:])')
+        else:
+            main(sys.argv[1:])
+    else:
         main(sys.argv[1:])