keep checksum of uncompressed metadata files in repomd.xml

diff --git a/dumpMetadata.py b/dumpMetadata.py
index 122a000e0ad1192e6bedd0abe10d00962dc342c8..80046ca0023e924c7af881aa6a90e395f29b6274 100644 (file)
--- a/dumpMetadata.py
+++ b/dumpMetadata.py
@@ -587,16 +587,17 @@ def otherXML(doc, node, rpmObj):
 def repoXML(node, cmds):
     """generate the repomd.xml file that stores the info on the other files"""
     sumtype = cmds['sumtype']
-    workfiles = [(cmds['otherfile'], 'other',),
-                 (cmds['filelistsfile'], 'filelists'),
-                 (cmds['primaryfile'], 'primary')]
-    
-    if cmds['groupfile'] is not None:
-        workfiles.append((cmds['groupfile'], 'group'))
+    workfiles = [(cmds['zotherfile'], 'other', cmds['otherfile']),
+                 (cmds['zfilelistsfile'], 'filelists', cmds['filelistsfile']),
+                 (cmds['zprimaryfile'], 'primary', cmds['primaryfile'])]
+
+    if cmds['zgroupfile'] is not None:
+        workfiles.append((cmds['zgroupfile'], 'group', cmds['groupfile']))
     
-    for (file, ftype) in workfiles:
+    for (file, ftype, un) in workfiles:
         csum = getChecksum(sumtype, os.path.join(cmds['tempdir'], file))
         timestamp = os.stat(os.path.join(cmds['tempdir'], file))[8]
+        uncsum = getChecksum(sumtype, os.path.join(cmds['tempdir'], un))
         data = node.newChild(None, 'data', None)
         data.newProp('type', ftype)
         location = data.newChild(None, 'location', None)
@@ -606,4 +607,6 @@ def repoXML(node, cmds):
         checksum = data.newChild(None, 'checksum', csum)
         checksum.newProp('type', sumtype)
         timestamp = data.newChild(None, 'timestamp', str(timestamp))
-            
+        unchecksum = data.newChild(None, 'uncompressed-checksum', uncsum)
+        unchecksum.newProp('type', sumtype)
+        

diff --git a/genpkgmetadata.py b/genpkgmetadata.py
index f63170df766f7d55382ce153b1e0b9f1eb2a70bb..69441346518e97cfcf64569eaed860d896f221e4 100755 (executable)
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -157,7 +157,8 @@ def parseArgs(args):
     cmds['verbose'] = 0
     cmds['excludes'] = []
     cmds['baseurl'] = None
-    cmds['groupfile'] = None
+    cmds['zgroupfile'] = None
+    cmds['groupfile'] = None    
     cmds['sumtype'] = 'md5'
     cmds['pretty'] = 0
 
@@ -227,7 +228,7 @@ def doPkgMetadata(cmds, ts):
     basens = baseroot.newNs('http://linux.duke.edu/metadata/common', None)
     baseroot.setNs(basens)
     basefilepath = os.path.join(cmds['tempdir'], cmds['primaryfile'])
-    basefile = _gzipOpen(basefilepath, 'w')
+    basefile = open(basefilepath, 'w')
     basefile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
     basefile.write('<metadata xmlns="http://linux.duke.edu/metadata/common">\n')
 
@@ -237,7 +238,7 @@ def doPkgMetadata(cmds, ts):
     filesns = filesroot.newNs('http://linux.duke.edu/metadata/filelists', None)
     filesroot.setNs(filesns)
     filelistpath = os.path.join(cmds['tempdir'], cmds['filelistsfile'])
-    flfile = _gzipOpen(filelistpath, 'w')    
+    flfile = open(filelistpath, 'w')    
     flfile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
     flfile.write('<filelists xmlns="http://linux.duke.edu/metadata/filelists">\n')
     
@@ -248,7 +249,7 @@ def doPkgMetadata(cmds, ts):
     otherns = otherroot.newNs('http://linux.duke.edu/metadata/other', None)
     otherroot.setNs(otherns)
     otherfilepath = os.path.join(cmds['tempdir'], cmds['otherfile'])
-    otherfile = _gzipOpen(otherfilepath, 'w')
+    otherfile = open(otherfilepath, 'w')
     otherfile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
     otherfile.write('<otherdata xmlns="http://linux.duke.edu/metadata/other">\n')
     
@@ -356,14 +357,33 @@ def doRepoMetadata(cmds):
         
     del repodoc
         
-   
+
+def compressMetadata(cmds):
+    """gzip compress the metadata files"""
+    compresslist = [('primaryfile', 'zprimaryfile'),
+                    ('filelistsfile', 'zfilelistsfile'),
+                    ('otherfile', 'zotherfile')]
+
+    for (un, com) in compresslist:
+        fn = os.path.join(cmds['tempdir'], cmds[un])
+        zfn = os.path.join(cmds['tempdir'], cmds[com])
+        zfo = _gzipOpen(zfn, mode='wb')
+        fo = open(fn, 'r')
+        zfo.write(fo.read())
+        fo.close()
+        zfo.close()
+
+                    
 
 def main(args):
     cmds, directory = parseArgs(args)
     #setup some defaults
-    cmds['primaryfile'] = 'primary.xml.gz'
-    cmds['filelistsfile'] = 'filelists.xml.gz'
-    cmds['otherfile'] = 'other.xml.gz'
+    cmds['primaryfile'] = 'primary.xml'
+    cmds['filelistsfile'] = 'filelists.xml'
+    cmds['otherfile'] = 'other.xml'
+    cmds['zprimaryfile'] = 'primary.xml.gz'
+    cmds['zfilelistsfile'] = 'filelists.xml.gz'
+    cmds['zotherfile'] = 'other.xml.gz'
     cmds['repomdfile'] = 'repomd.xml'
     cmds['tempdir'] = '.repodata'
     cmds['finaldir'] = 'repodata'
@@ -422,6 +442,9 @@ def main(args):
         # always clean up your messes
         os.chdir(curdir)
         raise
+
+    # compress it
+    compressMetadata(cmds)
     
     try:
         doRepoMetadata(cmds)
@@ -447,7 +470,8 @@ def main(args):
         sys.exit(1)
         
 
-    for file in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile']:
+    for file in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile',
+                 'zprimaryfile', 'zfilelistsfile', 'zotherfile']:
         oldfile = os.path.join(cmds['olddir'], cmds[file])
         if os.path.exists(oldfile):
             try:
@@ -464,7 +488,19 @@ def main(args):
         errorprint(_('Error was %s') % e)
         os.chdir(curdir)
         sys.exit(1)
-        
+
+    # get rid of the uncompressed files
+    for file in ['primaryfile', 'filelistsfile', 'otherfile']:
+        unfile = os.path.join(cmds['finaldir'], cmds[file])
+        if os.path.exists(unfile):
+            try:
+                os.remove(unfile)
+            except OSError, e:
+                errorprint(_('Could not remove uncompressed metadata file: %s') % unfile)
+                errorprint(_('Error was %s') % e)
+                os.chdir(curdir)
+                sys.exit(1)
+                
             
         
     # take us home mr. data