From 94078f7b0374f5593bce7eb39988bbefa5a78243 Mon Sep 17 00:00:00 2001 From: Gui Chen Date: Mon, 28 May 2012 10:24:18 +0800 Subject: [PATCH] pack images together and support compressed file format --pack-to is enabled for fs, loop, raw, and includes tar, tar.gz, tar.bz2, zip --compress-image is enabled for loop, raw, and includes gzip, bzip2 --pack-to is compatible with --taring-to --compress-image is compatible with --compress-disk-image Signed-off-by: Gui Chen --- mic/conf.py | 2 +- mic/creator.py | 20 +++++++----- mic/imager/baseimager.py | 54 +++++------------------------- mic/imager/fs.py | 27 +++++++++------ mic/imager/livecd.py | 9 ++++- mic/imager/liveusb.py | 6 ++++ mic/imager/loop.py | 76 ++++++++++++++++++------------------------- mic/imager/raw.py | 31 ++++++++++++------ mic/utils/misc.py | 68 ++++++++++++++++++++++++++++++++------ plugins/imager/loop_plugin.py | 18 +++++----- plugins/imager/raw_plugin.py | 10 ++++-- 11 files changed, 180 insertions(+), 141 deletions(-) diff --git a/mic/conf.py b/mic/conf.py index 47b5314..972c048 100644 --- a/mic/conf.py +++ b/mic/conf.py @@ -55,7 +55,7 @@ class ConfigMgr(object): "logfile": None, "record_pkgs": [], "rpmver": None, - "compress_disk_image": None, + "pack_to": None, "name_prefix": None, "proxy": None, "no_proxy": None, diff --git a/mic/creator.py b/mic/creator.py index d0560e3..e97ccf4 100644 --- a/mic/creator.py +++ b/mic/creator.py @@ -92,11 +92,16 @@ class Creator(cmdln.Cmdln): optparser.add_option('', '--runtime', type='string', dest='runtime', default=None, help='Specify runtime mode, avaiable: bootstrap') - optparser.add_option('', '--compress-disk-image', type='string', - dest='compress_disk_image', default=None, - help='Sets the disk image compression. NOTE: The ' - 'available values might depend on the used ' - 'filesystem type.') + # --taring-to is alias to --pack-to + optparser.add_option('', '--taring-to', type='string', + dest='pack_to', default=None, + help=SUPPRESS_HELP) + optparser.add_option('', '--pack-to', type='string', + dest='pack_to', default=None, + help='Pack the images together into the specified' + ' achive, extension supported: .zip, .tar, ' + '.tar.gz, .tar.bz2, etc. by default, .tar ' + 'will be used') optparser.add_option('', '--copy-kernel', action='store_true', dest='copy_kernel', help='Copy kernel files from image /boot directory' @@ -198,9 +203,8 @@ class Creator(cmdln.Cmdln): if self.options.runtime: configmgr.create['runtime'] = self.options.runtime - if self.options.compress_disk_image is not None: - configmgr.create['compress_disk_image'] = \ - self.options.compress_disk_image + if self.options.pack_to is not None: + configmgr.create['pack_to'] = self.options.pack_to if self.options.copy_kernel: configmgr.create['copy_kernel'] = self.options.copy_kernel diff --git a/mic/imager/baseimager.py b/mic/imager/baseimager.py index 2bc82e6..ce8160f 100644 --- a/mic/imager/baseimager.py +++ b/mic/imager/baseimager.py @@ -48,12 +48,6 @@ class BaseImageCreator(object): """ - # supported compression methods, and their corresponding cmd - zips = { - 'gz': 'gzip', - 'bz2': 'bzip2' - } - def __del__(self): self.cleanup() @@ -84,16 +78,12 @@ class BaseImageCreator(object): # If the kernel is save to the destdir when copy_kernel cmd is called. self._need_copy_kernel = False - # The compression method for disk image. - self._img_compression_method = None - if createopts: # Mapping table for variables that have different names. optmap = {"pkgmgr" : "pkgmgr_name", "outdir" : "destdir", "arch" : "target_arch", "local_pkgs_path" : "_local_pkgs_path", - "compress_disk_image" : "_img_compression_method", "copy_kernel" : "_need_copy_kernel", } @@ -119,6 +109,15 @@ class BaseImageCreator(object): # pending FEA: save log by default for --release + if self.pack_to: + if '@NAME@' in self.pack_to: + self.pack_to = self.pack_to.replace('@NAME@', self.name) + (tar, ext) = os.path.splitext(self.pack_to) + if ext in (".gz", ".bz2") and tar.endswith(".tar"): + ext = ".tar" + ext + if ext not in misc.pack_formats: + self.pack_to += ".tar" + self._dep_checks = ["ls", "bash", "cp", "echo", "modprobe", "passwd"] # Output image file names @@ -176,13 +175,6 @@ class BaseImageCreator(object): if not os.path.exists(self.cachedir): os.makedirs(self.cachedir) - if self._img_compression_method != None and \ - self._img_compression_method not in self.zips: - raise CreatorError("Given disk image compression method ('%s') is " - "not valid. Valid values are: %s." \ - % (self._img_compression_method, - ', '.join(self.zips.keys()))) - # # Properties @@ -1087,34 +1079,6 @@ class BaseImageCreator(object): if not os.path.exists(destdir): fs.makedirs(destdir) - if self._img_compression_method: - if not self._img_name: - raise CreatorError("Image name not set.") - - rc = None - img_location = os.path.join(self._outdir,self._img_name) - zipcmd = self.zips[self._img_compression_method] - - # confirm the existing of zip command - fs.find_binary_path(zipcmd) - - msger.info("Compressing %s with %s. Please wait ..." \ - % (img_location, zipcmd)) - rc = runner.show([zipcmd, "-f", img_location]) - if rc: - raise CreatorError("Failed to compress image %s with %s." \ - % (img_location, self._img_compression_method)) - - for bootimg in glob.glob(os.path.dirname(img_location) + \ - "/*-boot.bin"): - msger.info("Compressing %s with %s. Please wait..." \ - % (bootimg, zipcmd)) - rc = runner.show([zipcmd, "-f", bootimg]) - if rc: - raise CreatorError("Failed to compress image %s with " - "%s." \ - % (bootimg, - self._img_compression_method)) if self._recording_pkgs: self._save_recording_pkgs(destdir) diff --git a/mic/imager/fs.py b/mic/imager/fs.py index a548796..9cc4a70 100644 --- a/mic/imager/fs.py +++ b/mic/imager/fs.py @@ -47,7 +47,7 @@ class FsImageCreator(BaseImageCreator): if self._recording_pkgs: self._save_recording_pkgs(destdir) - if self._img_compression_method == None: + if not self.pack_to: fsdir = os.path.join(destdir, self.name) misc.check_space_pre_cp(self._instroot, destdir) @@ -60,10 +60,20 @@ class FsImageCreator(BaseImageCreator): self.outimage.append(fsdir) - elif self._img_compression_method == "tar.bz2": - dst = "%s/%s.tar.bz2" % (destdir, self.name) - msger.info("Creating %s (compressing %s with %s). Please wait..." \ - % (dst, self._instroot, self._img_compression_method)) + else: + (tar, comp) = os.path.splitext(self.pack_to) + try: + tarcreat = {'.tar': '-cf', + '.gz': '-czf', + '.bz2': '-cjf', + '.tgz': '-czf', + '.tbz': '-cjf'}[comp] + except KeyError: + raise CreatorError("Unsupported comression for this image type:" + " '%s', try '.tar', '.tar.gz', etc" % comp) + + dst = os.path.join(destdir, self.pack_to) + msger.info("Pack rootfs to %s. Please wait..." % dst) tar = find_binary_path('tar') tar_cmdline = [tar, "--numeric-owner", @@ -78,15 +88,12 @@ class FsImageCreator(BaseImageCreator): tar_cmdline.append("--exclude=%s" % (ignore_entry)) - tar_cmdline.extend(["-cjf", dst, "."]) + tar_cmdline.extend([tarcreat, dst, "."]) - rc = call(tar_cmdline) + rc = runner.show(tar_cmdline) if rc: raise CreatorError("Failed compress image with tar.bz2. " "Cmdline: %s" % (" ".join(tar_cmdline))) self.outimage.append(dst) - else: - raise CreatorError("Compression method '%s' not supported for 'fs' " - "image format." % (self._img_compression_method)) diff --git a/mic/imager/livecd.py b/mic/imager/livecd.py index 92d7e99..8854f62 100644 --- a/mic/imager/livecd.py +++ b/mic/imager/livecd.py @@ -20,7 +20,7 @@ import glob import shutil from mic import kickstart, msger -from mic.utils import fs_related, rpmmisc, runner +from mic.utils import fs_related, rpmmisc, runner, misc from mic.utils.errors import CreatorError from loop import LoopImageCreator @@ -286,6 +286,13 @@ class LiveImageCreatorBase(LoopImageCreator): self.__isodir + "/LiveOS/squashfs.img") self.__create_iso(self.__isodir) + + if self.pack_to: + isoimg = os.path.join(self._outdir, self.name + ".iso") + packimg = os.path.join(self._outdir, self.pack_to) + misc.packing(packimg, isoimg) + os.unlink(isoimg) + finally: shutil.rmtree(self.__isodir, ignore_errors = True) self.__isodir = None diff --git a/mic/imager/liveusb.py b/mic/imager/liveusb.py index e378d48..e46f122 100644 --- a/mic/imager/liveusb.py +++ b/mic/imager/liveusb.py @@ -294,6 +294,12 @@ class LiveUSBImageCreator(LiveCDImageCreator): self._create_usbimg(isodir) + if self.pack_to: + usbimg = os.path.join(self._outdir, self.name + ".usbimg") + packimg = os.path.join(self._outdir, self.pack_to) + misc.packing(packimg, usbimg) + os.unlink(usbimg) + finally: shutil.rmtree(isodir, ignore_errors = True) self._set_isodir(None) diff --git a/mic/imager/loop.py b/mic/imager/loop.py index f4f708b..fb2c1bd 100644 --- a/mic/imager/loop.py +++ b/mic/imager/loop.py @@ -98,7 +98,7 @@ class LoopImageCreator(BaseImageCreator): will be created as a separated loop image. """ - def __init__(self, creatoropts=None, pkgmgr=None, compress_to=None): + def __init__(self, creatoropts=None, pkgmgr=None, compress_image=None): """Initialize a LoopImageCreator instance. This method takes the same arguments as ImageCreator.__init__() @@ -109,18 +109,7 @@ class LoopImageCreator(BaseImageCreator): BaseImageCreator.__init__(self, creatoropts, pkgmgr) - if compress_to: - if '@NAME@' in compress_to: - compress_to = compress_to.replace('@NAME@', self.name) - - compress_imgdir_method = os.path.splitext(compress_to)[1] - if compress_imgdir_method in (".zip", ".tar"): - self.compress_imgdir_method = compress_imgdir_method[1:] - else: - self.compress_imgdir_method = "tar" - compress_to += ".tar" - - self.compress_to = compress_to + self.compress_image = compress_image self.__fslabel = None self.fslabel = self.name @@ -175,10 +164,7 @@ class LoopImageCreator(BaseImageCreator): else: self.__image_size = 0 - if compress_to: - self._img_name = self.compress_to - else: - self._img_name = self.name + ".img" + self._img_name = self.name + ".img" def _set_fstype(self, fstype): self.__fstype = fstype @@ -365,40 +351,38 @@ class LoopImageCreator(BaseImageCreator): item['loop'].cleanup() def _stage_final_image(self): - if self.compress_to: + if self.pack_to: self._resparse(0) + else: + self._resparse() - cfile_name = self.compress_to - mountfp_xml = os.path.splitext(cfile_name)[0] + ".xml" - - for item in self._instloops: - imgfile = os.path.join(self.__imgdir, item['name']) - if item['fstype'] == "ext4": - runner.show('/sbin/tune2fs ' - '-O ^huge_file,extents,uninit_bg %s ' \ - % imgfile) - - msger.info("Compress all loop images together to %s" % cfile_name) - dstfile = os.path.join(self._outdir, cfile_name) - if self.compress_imgdir_method == "tar": - misc.taring(dstfile, self.__imgdir) - elif self.compress_imgdir_method == "zip": - misc.ziping(dstfile, self.__imgdir) - else: - raise CreatorError("Unsupported compress type: %s" \ - % self.compress_imgdir_method) - # save mount points mapping file to xml - save_mountpoints(os.path.join(self._outdir, mountfp_xml), - self._instloops, - self.target_arch) + for item in self._instloops: + imgfile = os.path.join(self.__imgdir, item['name']) + if item['fstype'] == "ext4": + runner.show('/sbin/tune2fs -O ^huge_file,extents,uninit_bg %s ' + % imgfile) + if self.compress_image: + misc.compressing(imgfile, self.compress_image) + + if not self.pack_to: + for item in os.listdir(self.__imgdir): + shutil.move(os.path.join(self.__imgdir, item), + os.path.join(self._outdir, item)) + else: + msger.info("Pack all loop images together to %s" % self.pack_to) + dstfile = os.path.join(self._outdir, self.pack_to) + misc.packing(dstfile, self.__imgdir) + if self.pack_to: + mountfp_xml = os.path.splitext(self.pack_to)[0].rstrip('.tar') + ".xml" else: - self._resparse() - for item in self._instloops: - shutil.move(os.path.join(self.__imgdir, item['name']), - os.path.join(self._outdir, item['name'])) + mountfp_xml = self.name + ".xml" + # save mount points mapping file to xml + save_mountpoints(os.path.join(self._outdir, mountfp_xml), + self._instloops, + self.target_arch) def copy_attachment(self): if not hasattr(self, '_attachment') or not self._attachment: @@ -408,6 +392,8 @@ class LoopImageCreator(BaseImageCreator): msger.info("Copying attachment files...") for item in self._attachment: + if not os.path.exists(item): + continue dpath = os.path.join(self.__imgdir, os.path.basename(item)) msger.verbose("Copy attachment %s to %s" % (item, dpath)) shutil.copy(item, dpath) diff --git a/mic/imager/raw.py b/mic/imager/raw.py index 591cc53..b7b7686 100644 --- a/mic/imager/raw.py +++ b/mic/imager/raw.py @@ -22,7 +22,7 @@ import shutil from pykickstart.urlgrabber import progress from mic import kickstart, msger -from mic.utils import fs_related, runner +from mic.utils import fs_related, runner, misc from mic.utils.partitionedfs import PartitionedMount from mic.utils.errors import CreatorError, MountError @@ -36,12 +36,12 @@ class RawImageCreator(BaseImageCreator): subsequently be booted in a virtual machine or accessed with kpartx """ - def __init__(self, *args): + def __init__(self, creatoropts=None, pkgmgr=None, compress_image=None): """Initialize a ApplianceImageCreator instance. This method takes the same arguments as ImageCreator.__init__() """ - BaseImageCreator.__init__(self, *args) + BaseImageCreator.__init__(self, creatoropts, pkgmgr) self.__instloop = None self.__imgdir = None @@ -53,6 +53,7 @@ class RawImageCreator(BaseImageCreator): self.checksum = False self.appliance_version = None self.appliance_release = None + self.compress_image = compress_image #self.getsource = False #self.listpkg = False @@ -370,13 +371,23 @@ class RawImageCreator(BaseImageCreator): """ self._resparse() - msger.debug("moving disks to stage location") - for name in self.__disks.keys(): - src = "%s/%s-%s.raw" % (self.__imgdir, self.name,name) - self._img_name = "%s-%s.%s" % (self.name, name, self.__disk_format) - dst = "%s/%s" % (self._outdir, self._img_name) - msger.debug("moving %s to %s" % (src,dst)) - shutil.move(src,dst) + if self.compress_image: + for imgfile in os.listdir(self.__imgdir): + if imgfile.endswith('.raw') or imgfile.endswith('bin'): + misc.compressing(imgfile, self.compress_image) + + if self.pack_to: + dst = os.path.join(self._outdir, self.pack_to) + msger.info("Pack all raw images to %s" % dst) + misc.packing(dst, self.__imgdir) + else: + msger.debug("moving disks to stage location") + for name in self.__disks.keys(): + src = "%s/%s-%s.raw" % (self.__imgdir, self.name,name) + self._img_name = "%s-%s.%s" % (self.name, name, self.__disk_format) + dst = "%s/%s" % (self._outdir, self._img_name) + msger.debug("moving %s to %s" % (src,dst)) + shutil.move(src,dst) self._write_image_xml() def _write_image_xml(self): diff --git a/mic/utils/misc.py b/mic/utils/misc.py index 0c9cf10..7bdb81a 100644 --- a/mic/utils/misc.py +++ b/mic/utils/misc.py @@ -70,23 +70,71 @@ def extract_rpm(rpmfile, targetdir): os.chdir(olddir) -def taring(dstfile, targetdir): +def compressing(fpath, method): + comp_map = { + "gz": "gzip", + "bz2": "bzip2" + } + if method not in comp_map: + raise CreatorError("Unsupport compress format: %s, valid values: %s" + % (method, ','.join(comp_map.keys()))) + cmd = find_binary_path(comp_map[method]) + rc = runner.show([cmd, "-f", fpath]) + if rc: + raise CreatorError("Failed to %s file: %s" % (comp_map[method], fpath)) + +def taring(dstfile, target): import tarfile - wf = tarfile.open(dstfile, 'w') - for item in os.listdir(targetdir): - wf.add(os.path.join(targetdir, item), item) + ext = os.path.splitext(dstfile)[1] + comp = {".tar": None, + ".gz": "gz", # for .tar.gz + ".bz2": "bz2", # for .tar.bz2 + ".tgz": "gz", + ".tbz": "bz2"}[ext] + if not comp: + wf = tarfile.open(dstfile, 'w') + else: + wf = tarfile.open(dstfile, 'w:' + comp) + if os.path.isdir(target): + for item in os.listdir(target): + wf.add(os.path.join(target, item), item) + else: + wf.add(target, os.path.basename(target)) wf.close() -def ziping(dstfile, targetdir): +def ziping(dstfile, target): import zipfile wf = zipfile.ZipFile(dstfile, 'w', compression=zipfile.ZIP_DEFLATED) - for item in os.listdir(targetdir): - fpath = os.path.join(targetdir, item) - if not os.path.isfile(fpath): - continue - wf.write(fpath, item, zipfile.ZIP_DEFLATED) + if os.path.isdir(target): + for item in os.listdir(target): + fpath = os.path.join(target, item) + if not os.path.isfile(fpath): + continue + wf.write(fpath, item, zipfile.ZIP_DEFLATED) + else: + wf.write(target, os.path.basename(target), zipfile.ZIP_DEFLATED) wf.close() +pack_formats = { + ".tar": taring, + ".tar.gz": taring, + ".tar.bz2": taring, + ".tgz": taring, + ".tbz": taring, + ".zip": ziping, +} + +def packing(dstfile, target): + (base, ext) = os.path.splitext(dstfile) + if ext in (".gz", ".bz2") and base.endswith(".tar"): + ext = ".tar" + ext + if ext not in pack_formats: + raise CreatorError("Unsupport pack format: %s, valid values: %s" + % (ext, ','.join(pack_formats.keys()))) + func = pack_formats[ext] + # func should be callable + func(dstfile, target) + def human_size(size): """Return human readable string for Bytes size """ diff --git a/plugins/imager/loop_plugin.py b/plugins/imager/loop_plugin.py index f4f480c..de9c431 100644 --- a/plugins/imager/loop_plugin.py +++ b/plugins/imager/loop_plugin.py @@ -30,12 +30,12 @@ class LoopPlugin(ImagerPlugin): name = 'loop' @classmethod - @cmdln.option("--taring-to", dest="compress_to", type='string', - default=None, help="same with '--compress-to'") - @cmdln.option("--compress-to", dest="compress_to", type='string', - default=None, help="Specify compress filename for all image " - "output, compress type decided by file extension, '.zip' for " - "zip format, '.tar' for tar format, default is tar format") + @cmdln.option("--compress-disk-image", dest="compress_image", type='choice', + choices=("gz", "bz2"), default=None, + help="Same with --compress-image") + @cmdln.option("--compress-image", dest="compress_image", type='choice', + choices=("gz", "bz2"), default=None, + help="Compress all loop images wiht 'gz' or 'bz2'") def do_create(self, subcmd, opts, *args): """${cmd_name}: create loop image @@ -92,14 +92,14 @@ class LoopPlugin(ImagerPlugin): if creatoropts['runtime']: rt_util.runmic_in_runtime(creatoropts['runtime'], creatoropts, ksconf, None) - creator = LoopImageCreator(creatoropts, pkgmgr, opts.compress_to) + creator = LoopImageCreator(creatoropts, pkgmgr, opts.compress_image) if len(recording_pkgs) > 0: creator._recording_pkgs = recording_pkgs if creatoropts['release'] is None: - if opts.compress_to: - imagefile = "%s" % os.path.join(creator.destdir, creator.compress_to) + if creatoropts['pack_to']: + imagefile = "%s" % os.path.join(creator.destdir, creator.pack_to) else: imagefile = "%s.img" % os.path.join(creator.destdir, creator.name) diff --git a/plugins/imager/raw_plugin.py b/plugins/imager/raw_plugin.py index f0d02bb..6fbb35f 100644 --- a/plugins/imager/raw_plugin.py +++ b/plugins/imager/raw_plugin.py @@ -21,7 +21,7 @@ import re import tempfile from mic import chroot, msger, rt_util -from mic.utils import misc, fs_related, errors, runner +from mic.utils import misc, fs_related, errors, runner, cmdln from mic.conf import configmgr from mic.plugin import pluginmgr from mic.utils.partitionedfs import PartitionedMount @@ -33,6 +33,12 @@ class RawPlugin(ImagerPlugin): name = 'raw' @classmethod + @cmdln.option("--compress-disk-image", dest="compress_image", type='choice', + choices=("gz", "bz2"), default=None, + help="Same with --compress-image") + @cmdln.option("--compress-image", dest="compress_image", type='choice', + choices=("gz", "bz2"), default = None, + help="Compress all raw images before package") def do_create(self, subcmd, opts, *args): """${cmd_name}: create raw image @@ -83,7 +89,7 @@ class RawPlugin(ImagerPlugin): if creatoropts['runtime']: rt_util.runmic_in_runtime(creatoropts['runtime'], creatoropts, ksconf, None) - creator = raw.RawImageCreator(creatoropts, pkgmgr) + creator = raw.RawImageCreator(creatoropts, pkgmgr, opts.compress_image) if len(recording_pkgs) > 0: creator._recording_pkgs = recording_pkgs -- 2.7.4