From: JF Ding Date: Wed, 7 Sep 2011 00:44:28 +0000 (+0900) Subject: replace most of subprocess calls by new runner apis X-Git-Tag: 0.1~147 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=8762fbec9a3044138d6de341c9aa6608a8259a6d;p=tools%2Fmic.git replace most of subprocess calls by new runner apis only several exceptions remained: chroot ones pipe chained ones --- diff --git a/mic/chroot.py b/mic/chroot.py index 35abc0b..db0d55f 100644 --- a/mic/chroot.py +++ b/mic/chroot.py @@ -18,14 +18,12 @@ # from __future__ import with_statement -import os, sys +import os import shutil import subprocess -import mic.utils.fs_related as fs_related -import mic.utils.misc as misc -import mic.utils.errors as errors from mic import msger +from mic.utils import misc, errors, runner, fs_related chroot_lockfd = -1 chroot_lock = "" @@ -259,11 +257,7 @@ def chroot(chrootdir, bindmounts = None, execute = "/bin/bash"): if not os.path.exists(ftc): continue - filecmd = misc.find_binary_path("file") - - for line in subprocess.Popen([filecmd, ftc], - stdout=subprocess.PIPE, - stderr=dev_null).communicate()[0].strip().splitlines(): + for line in runner.outs(['file', ftc]).splitlines(): if 'ARM' in line: qemu_emulator = misc.setup_qemu_emulator(chrootdir, "arm") architecture_found = True @@ -285,8 +279,8 @@ def chroot(chrootdir, bindmounts = None, execute = "/bin/bash"): globalmounts = setup_chrootenv(chrootdir, bindmounts) subprocess.call(execute, preexec_fn = mychroot, shell=True) - except OSError, (err, msg): - raise errors.CreatorError("Failed to chroot: %s" % msg) + except OSError, err: + raise errors.CreatorError("chroot err: %s" % str(err)) finally: cleanup_chrootenv(chrootdir, bindmounts, globalmounts) diff --git a/mic/imager/baseimager.py b/mic/imager/baseimager.py index fffbe07..5cfda0d 100644 --- a/mic/imager/baseimager.py +++ b/mic/imager/baseimager.py @@ -2,6 +2,7 @@ # creator.py : ImageCreator and LoopImageCreator base classes # # Copyright 2007, Red Hat Inc. +# Copyright 2009, 2010, 2011 Intel, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,12 +28,10 @@ import glob import rpm -from mic.utils.errors import CreatorError -from mic.utils.misc import get_filesystem_avail, is_statically_linked,setup_qemu_emulator, create_release -from mic.utils.fs_related import find_binary_path, makedirs, BindChrootMount -from mic.utils import rpmmisc, runner from mic import kickstart from mic import msger +from mic.utils.errors import CreatorError +from mic.utils import misc, rpmmisc, runner, fs_related as fs class BaseImageCreator(object): """Installs a system to a chroot directory. @@ -133,9 +132,9 @@ class BaseImageCreator(object): # make sure the specified tmpdir and cachedir exist if not os.path.exists(self.tmpdir): - makedirs(self.tmpdir) + os.makedirs(self.tmpdir) if not os.path.exists(self.cachedir): - makedirs(self.cachedir) + os.makedirs(self.cachedir) def set_target_arch(self, arch): if arch not in rpmmisc.arches: @@ -147,7 +146,7 @@ class BaseImageCreator(object): if dep == "extlinux": self._dep_checks.remove(dep) - if not os.path.exists("/usr/bin/qemu-arm") or not is_statically_linked("/usr/bin/qemu-arm"): + if not os.path.exists("/usr/bin/qemu-arm") or not misc.is_statically_linked("/usr/bin/qemu-arm"): self._dep_checks.append("qemu-arm-static") if os.path.exists("/proc/sys/vm/vdso_enabled"): @@ -161,7 +160,6 @@ class BaseImageCreator(object): return True - def __del__(self): self.cleanup() @@ -272,7 +270,7 @@ class BaseImageCreator(object): # save package name list anyhow if not os.path.exists(destdir): - makedirs(destdir) + os.makedirs(destdir) namefile = os.path.join(destdir, self.name + '-pkgs.txt') f = open(namefile, "w") @@ -580,7 +578,7 @@ class BaseImageCreator(object): self.cachedir = cachedir else: self.cachedir = self.__builddir + "/yum-cache" - makedirs(self.cachedir) + fs.makedirs(self.cachedir) return self.cachedir def __sanity_check(self): @@ -622,7 +620,6 @@ class BaseImageCreator(object): os.symlink(src, self._instroot + dest) os.umask(origumask) - def mount(self, base_on = None, cachedir = None): """Setup the target filesystem in preparation for an install. @@ -643,16 +640,16 @@ class BaseImageCreator(object): """ self.__ensure_builddir() - makedirs(self._instroot) - makedirs(self._outdir) + fs.makedirs(self._instroot) + fs.makedirs(self._outdir) self._mount_instroot(base_on) for d in ("/dev/pts", "/etc", "/boot", "/var/log", "/var/cache/yum", "/sys", "/proc", "/usr/bin"): - makedirs(self._instroot + d) + fs.makedirs(self._instroot + d) if self.target_arch and self.target_arch.startswith("arm"): - self.qemu_emulator = setup_qemu_emulator(self._instroot, self.target_arch) + self.qemu_emulator = misc.setup_qemu_emulator(self._instroot, self.target_arch) self.get_cachedir(cachedir) @@ -660,8 +657,7 @@ class BaseImageCreator(object): for (f, dest) in [("/sys", None), ("/proc", None), ("/proc/sys/fs/binfmt_misc", None), ("/dev/pts", None), (self.get_cachedir(), "/var/cache/yum")]: - self.__bindmounts.append(BindChrootMount(f, self._instroot, dest)) - + self.__bindmounts.append(fs.BindChrootMount(f, self._instroot, dest)) self._do_bindmounts() @@ -674,7 +670,7 @@ class BaseImageCreator(object): self.__write_fstab() # get size of available space in 'instroot' fs - self._root_fs_avail = get_filesystem_avail(self._instroot) + self._root_fs_avail = misc.get_filesystem_avail(self._instroot) def unmount(self): """Unmounts the target filesystem. @@ -695,9 +691,7 @@ class BaseImageCreator(object): except OSError: pass - self._undo_bindmounts() - self._unmount_instroot() def cleanup(self): @@ -791,7 +785,6 @@ class BaseImageCreator(object): """ - # initialize pkg list to install if self.ks: self.__sanity_check() @@ -898,7 +891,7 @@ class BaseImageCreator(object): return None gpgkeydir = "/etc/pki/rpm-gpg" - makedirs(self._instroot + gpgkeydir) + fs.makedirs(self._instroot + gpgkeydir) for repo in repodata: if repo["repokey"]: repokey = gpgkeydir + "/RPM-GPG-KEY-%s" % repo["name"] @@ -956,10 +949,8 @@ class BaseImageCreator(object): """ Generate md5sum if /usr/bin/md5sum is available """ if os.path.exists("/usr/bin/md5sum"): - p = subprocess.Popen(["/usr/bin/md5sum", "-b", image_name], - stdout=subprocess.PIPE) - (md5sum, errorstr) = p.communicate() - if p.returncode != 0: + (rc, md5sum) = runner.runtool(["/usr/bin/md5sum", "-b", image_name]) + if rc != 0: msger.warning("Can't generate md5sum for image %s" % image_name) else: pattern = re.compile("\*.*$") @@ -984,14 +975,14 @@ class BaseImageCreator(object): self._stage_final_image() if not os.path.exists(destdir): - makedirs(destdir) + fs.makedirs(destdir) if self.__img_compression_method: if not self._img_name: raise CreatorError("Image name not set.") rc = None img_location = os.path.join(self._outdir,self._img_name) if self.__img_compression_method == "bz2": - bzip2 = find_binary_path('bzip2') + bzip2 = fs.find_binary_path('bzip2') msger.info("Compressing %s with bzip2. Please wait..." % img_location) rc = runner.show([bzip2, "-f", img_location]) if rc: @@ -1009,7 +1000,7 @@ class BaseImageCreator(object): if self.image_format in ("raw", "vmdk", "vdi", "nand", "mrstnand"): destdir = os.path.join(destdir, "%s-%s" % (self.name, self.image_format)) msger.debug("creating destination dir: %s" % destdir) - makedirs(destdir) + fs.makedirs(destdir) # Ensure all data is flushed to _outdir runner.quiet('sync') @@ -1030,7 +1021,7 @@ class BaseImageCreator(object): def check_depend_tools(self): for tool in self._dep_checks: - find_binary_path(tool) + fs.find_binary_path(tool) def package_output(self, image_format, destdir = ".", package="none"): if not package or package == "none": @@ -1057,18 +1048,17 @@ class BaseImageCreator(object): else: os.remove(file) - tar.close() '''All the file in outimage has been packaged into tar.* file''' self.outimage = [dst] def release_output(self, config, destdir, name, release): - self.outimage = create_release(config, destdir, name, self.outimage, release) + self.outimage = misc.create_release(config, destdir, name, self.outimage, release) def save_kernel(self, destdir): if not os.path.exists(destdir): - makedirs(destdir) + os.makedirs(destdir) for kernel in glob.glob("%s/boot/vmlinuz-*" % self._instroot): kernelfilename = "%s/%s-%s" % (destdir, self.name, os.path.basename(kernel)) shutil.copy(kernel, kernelfilename) @@ -1087,5 +1077,3 @@ class BaseImageCreator(object): def get_pkg_manager(self, recording_pkgs=None): return self.pkgmgr(creator = self, recording_pkgs = recording_pkgs) - - diff --git a/mic/utils/fs_related.py b/mic/utils/fs_related.py index 544caca..33a69bc 100644 --- a/mic/utils/fs_related.py +++ b/mic/utils/fs_related.py @@ -21,7 +21,6 @@ import os import sys import errno import stat -import subprocess import random import string import time @@ -109,26 +108,22 @@ def resize2fs(fs, size): resize2fs = find_binary_path("resize2fs") return runner.quiet([resize2fs, fs, "%sK" % (size / 1024,)]) -def my_fuser(file): - ret = False +def my_fuser(fp): fuser = find_binary_path("fuser") - if not os.path.exists(file): - return ret + if not os.path.exists(fp): + return False - dev_null = os.open("/dev/null", os.O_WRONLY) - rc = runner.quiet([fuser, "-s", file]) + rc = runner.quiet([fuser, "-s", fp]) if rc == 0: - fuser_proc = subprocess.Popen([fuser, file], stdout=subprocess.PIPE, stderr=dev_null) - pids = fuser_proc.communicate()[0].strip().split() - for pid in pids: + for pid in runner.outs([fuser, fp]).split(): fd = open("/proc/%s/cmdline" % pid, "r") cmdline = fd.read() fd.close() if cmdline[:-1] == "/bin/bash": - ret = True - break - os.close(dev_null) - return ret + return True + + # not found + return False class BindChrootMount: """Represents a bind mount of a directory into a chroot.""" @@ -146,18 +141,12 @@ class BindChrootMount: self.umountcmd = find_binary_path("umount") def ismounted(self): - ret = False - dev_null = os.open("/dev/null", os.O_WRONLY) - catcmd = find_binary_path("cat") - args = [ catcmd, "/proc/mounts" ] - proc_mounts = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=dev_null) - outputs = proc_mounts.communicate()[0].strip().split("\n") - for line in outputs: - if line.split()[1] == os.path.abspath(self.dest): - ret = True - break - os.close(dev_null) - return ret + with open('/proc/mounts') as f: + for line in f: + if line.split()[1] == os.path.abspath(self.dest): + return True + + return False def has_chroot_instance(self): lock = os.path.join(self.root, ".chroot.lock") @@ -209,11 +198,8 @@ class LoopbackMount: if self.losetup: return - losetupProc = subprocess.Popen([self.losetupcmd, "-f"], - stdout=subprocess.PIPE) - losetupOutput = losetupProc.communicate()[0] - - if losetupProc.returncode: + rc, losetupOutput = runner.runtool([self.losetupcmd, "-f"]) + if rc != 0: raise MountError("Failed to allocate loop device for '%s'" % self.lofile) @@ -331,11 +317,8 @@ class LoopbackDisk(Disk): if self.device is not None: return - losetupProc = subprocess.Popen([self.losetupcmd, "-f"], - stdout=subprocess.PIPE) - losetupOutput = losetupProc.communicate()[0] - - if losetupProc.returncode: + rc, losetupOutput = runner.runtool([self.losetupcmd, "-f"]) + if rc != 0: raise MountError("Failed to allocate loop device for '%s'" % self.lofile) @@ -508,13 +491,7 @@ class ExtDiskMount(DiskMount): if rc != 0: raise MountError("Error creating %s filesystem on disk %s" % (self.fstype, self.disk.device)) - dev_null = os.open("/dev/null", os.O_WRONLY) - try: - out = subprocess.Popen([self.dumpe2fs, '-h', self.disk.device], - stdout = subprocess.PIPE, - stderr = dev_null).communicate()[0] - finally: - os.close(dev_null) + out = runner.outs([self.dumpe2fs, '-h', self.disk.device]) self.uuid = self.__parse_field(out, "Filesystem UUID") msger.debug("Tuning filesystem on %s" % self.disk.device) @@ -558,15 +535,8 @@ class ExtDiskMount(DiskMount): runner.show(["/sbin/e2fsck", "-f", "-y", self.disk.lofile]) def __get_size_from_filesystem(self): - dev_null = os.open("/dev/null", os.O_WRONLY) - try: - out = subprocess.Popen([self.dumpe2fs, '-h', self.disk.lofile], - stdout = subprocess.PIPE, - stderr = dev_null).communicate()[0] - finally: - os.close(dev_null) - - return int(self.__parse_field(out, "Block count")) * self.blocksize + return int(self.__parse_field(runner.outs([self.dumpe2fs, '-h', self.disk.lofile]), + "Block count")) * self.blocksize def __resize_to_minimal(self): self.__fsck() @@ -719,15 +689,7 @@ class BtrfsDiskMount(DiskMount): if rc != 0: raise MountError("Error creating %s filesystem on disk %s" % (self.fstype,self.disk.device)) - dev_null = os.open("/dev/null", os.O_WRONLY) - try: - out = subprocess.Popen([self.blkidcmd, self.disk.device], - stdout = subprocess.PIPE, - stderr = dev_null).communicate()[0] - finally: - os.close(dev_null) - - self.uuid = self.__parse_field(out, "UUID") + self.uuid = self.__parse_field(runner.outs([self.blkidcmd, self.disk.device]), "UUID") def __resize_filesystem(self, size = None): current_size = os.stat(self.disk.lofile)[stat.ST_SIZE] @@ -840,14 +802,6 @@ class DeviceMapperSnapshot(object): if not self.__created: return 0 - dev_null = os.open("/dev/null", os.O_WRONLY) - try: - out = subprocess.Popen([self.dmsetupcmd, "status", self.__name], - stdout = subprocess.PIPE, - stderr = dev_null).communicate()[0] - finally: - os.close(dev_null) - # # dmsetup status on a snapshot returns e.g. # "0 8388608 snapshot 416/1048576" @@ -855,6 +809,7 @@ class DeviceMapperSnapshot(object): # "A B snapshot C/D" # where C is the number of 512 byte sectors in use # + out = runner.outs([self.dmsetupcmd, "status", self.__name]) try: return int((out.split()[3]).split('/')[0]) * 512 except ValueError: diff --git a/mic/utils/misc.py b/mic/utils/misc.py index a0049a5..4285cd3 100644 --- a/mic/utils/misc.py +++ b/mic/utils/misc.py @@ -18,7 +18,6 @@ import os import sys -import subprocess import tempfile import re import shutil @@ -39,6 +38,7 @@ xmlparse = cElementTree.parse from errors import * from fs_related import * from proxy import get_proxy_for +import runner from mic import msger @@ -85,12 +85,7 @@ def get_image_type(path): if file_header[0:len(vdi_flag)] == vdi_flag: return maptab["vdi"] - dev_null = os.open("/dev/null", os.O_WRONLY) - filecmd = find_binary_path("file") - args = [ filecmd, path ] - file = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=dev_null) - output = file.communicate()[0] - os.close(dev_null) + output = runner.outs(['file', path]) isoptn = re.compile(r".*ISO 9660 CD-ROM filesystem.*(bootable).*") usbimgptn = re.compile(r".*x86 boot sector.*active.*") rawptn = re.compile(r".*x86 boot sector.*") @@ -111,23 +106,16 @@ def get_image_type(path): def get_file_size(file): """Return size in MB unit""" - du = find_binary_path("du") - dev_null = os.open("/dev/null", os.O_WRONLY) - duProc = subprocess.Popen([du, "-s", "-b", "-B", "1M", file], - stdout=subprocess.PIPE, stderr=dev_null) - duOutput = duProc.communicate()[0] - if duProc.returncode: + rc, duOutput = runner.runtool(['du', "-s", "-b", "-B", "1M", file]) + if rc != 0: raise CreatorError("Failed to run %s" % du) size1 = int(duOutput.split()[0]) - duProc = subprocess.Popen([du, "-s", "-B", "1M", file], - stdout=subprocess.PIPE, stderr=dev_null) - duOutput = duProc.communicate()[0] - if duProc.returncode: + rc, duOutput = runner.runtool(['du', "-s", "-B", "1M", file]) + if rc != 0: raise CreatorError("Failed to run %s" % du) size2 = int(duOutput.split()[0]) - os.close(dev_null) if size1 > size2: return size1 else: @@ -151,7 +139,7 @@ def convert_image(srcimg, srcfmt, dstimg, dstfmt): else: raise CreatorError("Invalid soure image format: %s" % srcfmt) - rc = subprocess.call(argv) + rc = runner.show(argv) if rc == 0: msger.debug("convert successful") if rc != 0: @@ -161,7 +149,7 @@ def uncompress_squashfs(squashfsimg, outdir): """Uncompress file system from squshfs image""" unsquashfs = find_binary_path("unsquashfs") args = [ unsquashfs, "-d", outdir, squashfsimg ] - rc = subprocess.call(args) + rc = runner.show(args) if (rc != 0): raise SquashfsError("Failed to uncompress %s." % squashfsimg) @@ -215,11 +203,11 @@ def get_uncompressed_data_from_url(url, filename, proxies): if filename.endswith(".gz"): suffix = ".gz" gunzip = find_binary_path('gunzip') - subprocess.call([gunzip, "-f", filename]) + runner.show([gunzip, "-f", filename]) elif filename.endswith(".bz2"): suffix = ".bz2" bunzip2 = find_binary_path('bunzip2') - subprocess.call([bunzip2, "-f", filename]) + runner.show([bunzip2, "-f", filename]) if suffix: filename = filename.replace(suffix, "") return filename @@ -416,6 +404,8 @@ def get_source_name(pkg, repometadata): return None def get_release_no(repometadata, distro="meego"): + import subprocess + cpio = find_binary_path("cpio") rpm2cpio = find_binary_path("rpm2cpio") release_pkg = get_package("%s-release" % distro, repometadata) @@ -559,25 +549,16 @@ def get_pkglist_in_comps(group, comps): return pkglist def is_statically_linked(binary): - ret = False - dev_null = os.open("/dev/null", os.O_WRONLY) - filecmd = find_binary_path("file") - args = [ filecmd, binary ] - file = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=dev_null) - output = file.communicate()[0] - os.close(dev_null) - if output.find(", statically linked, ") > 0: - ret = True - return ret + return ", statically linked, " in runner.outs(['file', binary]) def setup_qemu_emulator(rootdir, arch): # mount binfmt_misc if it doesn't exist if not os.path.exists("/proc/sys/fs/binfmt_misc"): modprobecmd = find_binary_path("modprobe") - subprocess.call([modprobecmd, "binfmt_misc"]) + runner.show([modprobecmd, "binfmt_misc"]) if not os.path.exists("/proc/sys/fs/binfmt_misc/register"): mountcmd = find_binary_path("mount") - subprocess.call([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"]) + runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"]) # qemu_emulator is a special case, we can't use find_binary_path # qemu emulator should be a statically-linked executable file @@ -592,7 +573,8 @@ def setup_qemu_emulator(rootdir, arch): # disable selinux, selinux will block qemu emulator to run if os.path.exists("/usr/sbin/setenforce"): - subprocess.call(["/usr/sbin/setenforce", "0"]) + msger.info('Try to disable selinux') + runner.show(["/usr/sbin/setenforce", "0"]) node = "/proc/sys/fs/binfmt_misc/arm" if is_statically_linked(qemu_emulator) and os.path.exists(node): @@ -632,7 +614,7 @@ def create_release(config, destdir, name, outimages, release): shutil.rmtree(thatsubdir, ignore_errors = True) """ Create release directory and files """ - os.system ("cp %s %s/%s.ks" % (config, destdir, name)) + runner.show("cp %s %s/%s.ks" % (config, destdir, name)) # When building a release we want to make sure the .ks # file generates the same build even when --release= is not used. fd = open(config, "r") @@ -645,11 +627,10 @@ def create_release(config, destdir, name, outimages, release): outimages.append("%s/%s.ks" % (destdir,name)) # Using system + mv, because of * in filename. - os.system ("mv %s/*-pkgs.txt %s/%s.packages" % (destdir, destdir, name)) + runner.show("mv %s/*-pkgs.txt %s/%s.packages" % (destdir, destdir, name)) outimages.append("%s/%s.packages" % (destdir,name)) - d = os.listdir(destdir) - for f in d: + for f in os.listdir(destdir): if f.endswith(".iso"): ff = f.replace(".iso", ".img") os.rename("%s/%s" %(destdir, f ), "%s/%s" %(destdir, ff)) @@ -659,19 +640,17 @@ def create_release(config, destdir, name, outimages, release): os.rename("%s/%s" %(destdir, f ), "%s/%s" %(destdir, ff)) outimages.append("%s/%s" %(destdir, ff)) - fd = open(destdir + "/MANIFEST", "w") - d = os.listdir(destdir) - for f in d: - if f == "MANIFEST": - continue - if os.path.exists("/usr/bin/md5sum"): - p = subprocess.Popen(["/usr/bin/md5sum", "-b", "%s/%s" %(destdir, f )], - stdout=subprocess.PIPE) - (md5sum, errorstr) = p.communicate() - if p.returncode != 0: + if os.path.exists("/usr/bin/md5sum"): + fd = open(destdir + "/MANIFEST", "w") + for f in os.listdir(destdir): + if f == "MANIFEST": + continue + + rc, md5sum = runner.runtool(["/usr/bin/md5sum", "-b", "%s/%s" %(destdir, f )]) + if rc != 0: msger.warning("Can't generate md5sum for image %s/%s" %(destdir, f )) else: - md5sum = md5sum.split(" ")[0] + md5sum = md5sum.lstrip().split()[0] fd.write(md5sum+" "+f+"\n") outimages.append("%s/MANIFEST" % destdir) diff --git a/mic/utils/partitionedfs.py b/mic/utils/partitionedfs.py index 2367471..4706b75 100644 --- a/mic/utils/partitionedfs.py +++ b/mic/utils/partitionedfs.py @@ -5,6 +5,8 @@ # Copyright 2008, Daniel P. Berrange # Copyright 2008, David P. Huff # +# Copyright 2009, 2010, 2011 Intel, Inc. +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. @@ -19,14 +21,11 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. import os -import glob -import shutil -import subprocess -import time -from mic.utils.errors import * -from mic.utils.fs_related import * from mic import msger +from mic.utils import runner +from mic.utils.errors import MountError +from mic.utils.fs_related import * class PartitionedMount(Mount): def __init__(self, disks, mountdir, skipformat = False): @@ -117,11 +116,11 @@ class PartitionedMount(Mount): part_cmd.extend(["%d" % start, "%d" % end]) msger.debug(part_cmd) - p1 = subprocess.Popen(part_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out = p1.communicate()[0].strip() + rc, out = runner.runtool(part_cmd, catch=3) + out = out.strip() if out: msger.debug('"parted" output: %s' % out) - return p1.returncode + return rc def __format_disks(self): msger.debug("Assigning partitions to disks") @@ -163,13 +162,12 @@ class PartitionedMount(Mount): for dev in self.disks.keys(): d = self.disks[dev] msger.debug("Initializing partition table for %s" % (d['disk'].device)) - p1 = subprocess.Popen([self.parted, "-s", d['disk'].device, "mklabel", "msdos"], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out = p1.communicate()[0].strip() + rc, out = runner.runtool([self.parted, "-s", d['disk'].device, "mklabel", "msdos"], catch=3) + out = out.strip() if out: msger.debug('"parted" output: %s' % out) - if p1.returncode != 0: + if rc != 0: # NOTE: We don't throw exception when return code is not 0, because # parted always fails to reload part table with loop devices. # This prevents us from distinguishing real errors based on return code. @@ -212,35 +210,28 @@ class PartitionedMount(Mount): msger.debug("Setting boot flag for partition '%s' on disk '%s'." % (p['num'],d['disk'].device)) boot_cmd = [self.parted, "-s", d['disk'].device, "set", "%d" % p['num'], "boot", "on"] msger.debug(boot_cmd) - p1 = subprocess.Popen(boot_cmd, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - (out,err) = p1.communicate() - msger.debug(out) + rc = runner.show(boot_cmd) - if p1.returncode != 0: + if rc != 0: # NOTE: We don't throw exception when return code is not 0, because # parted always fails to reload part table with loop devices. # This prevents us from distinguishing real errors based on return code. - msger.debug("WARNING: parted returned '%s' instead of 0 when adding boot flag for partition '%s' disk '%s'." % (p1.returncode,p['num'],d['disk'].device)) + msger.warning("parted returned '%s' instead of 0 when adding boot flag for partition '%s' disk '%s'." % (rc,p['num'],d['disk'].device)) def __map_partitions(self): """Load it if dm_snapshot isn't loaded""" load_module("dm_snapshot") - dev_null = os.open("/dev/null", os.O_WRONLY) for dev in self.disks.keys(): d = self.disks[dev] if d['mapped']: continue msger.debug("Running kpartx on %s" % d['disk'].device ) - kpartx = subprocess.Popen([self.kpartx, "-l", "-v", d['disk'].device], - stdout=subprocess.PIPE, stderr=dev_null) + rc, kpartxOutput = runner.runtool([self.kpartx, "-l", "-v", d['disk'].device]) + kpartxOutput = kpartxOutput.splitlines() - kpartxOutput = kpartx.communicate()[0].strip().split("\n") - - if kpartx.returncode: - os.close(dev_null) + if rc != 0: raise MountError("Failed to query partition mapping for '%s'" % d['disk'].device) @@ -254,7 +245,6 @@ class PartitionedMount(Mount): # our expectation. If it doesn't, someone broke the code # further up if len(kpartxOutput) != d['numpart']: - os.close(dev_null) raise MountError("Unexpected number of partitions from kpartx: %d != %d" % (len(kpartxOutput), d['numpart'])) @@ -277,27 +267,19 @@ class PartitionedMount(Mount): os.symlink(mapperdev, loopdev) msger.debug("Adding partx mapping for %s" % d['disk'].device) - p1 = subprocess.Popen([self.kpartx, "-v", "-a", d['disk'].device], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + rc = runner.show([self.kpartx, "-v", "-a", d['disk'].device]) - (out,err) = p1.communicate() - msger.debug(out) - - if p1.returncode != 0: + if rc != 0: # Make sure that the device maps are also removed on error case. # The d['mapped'] isn't set to True if the kpartx fails so # failed mapping will not be cleaned on cleanup either. - subprocess.call([self.kpartx, "-d", d['disk'].device], - stdout=dev_null, stderr=dev_null) - os.close(dev_null) + runner.quiet([self.kpartx, "-d", d['disk'].device]) raise MountError("Failed to map partitions for '%s'" % d['disk'].device) - d['mapped'] = True - os.close(dev_null) + d['mapped'] = True def __unmap_partitions(self): - dev_null = os.open("/dev/null", os.O_WRONLY) for dev in self.disks.keys(): d = self.disks[dev] if not d['mapped']: @@ -310,16 +292,12 @@ class PartitionedMount(Mount): self.partitions[pnum]['device'] = None msger.debug("Unmapping %s" % d['disk'].device) - rc = subprocess.call([self.kpartx, "-d", d['disk'].device], - stdout=dev_null, stderr=dev_null) + rc = runner.quiet([self.kpartx, "-d", d['disk'].device]) if rc != 0: - os.close(dev_null) raise MountError("Failed to unmap partitions for '%s'" % d['disk'].device) d['mapped'] = False - os.close(dev_null) - def __calculate_mountorder(self): msger.debug("Calculating mount order") @@ -367,15 +345,17 @@ class PartitionedMount(Mount): if not self.btrfscmd: self.btrfscmd=find_binary_path("btrfs") argv = [ self.btrfscmd, "subvolume", "list", rootpath ] - p1 = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - (out,err) = p1.communicate() + + rc, out = runner.runtool(argv) msger.debug(out) - if p1.returncode != 0: - raise MountError("Failed to get subvolume id from %s', return code: %d." % (rootpath, p1.returncode)) + + if rc != 0: + raise MountError("Failed to get subvolume id from %s', return code: %d." % (rootpath, rc)) + subvolid = -1 - for line in out.split("\n"): + for line in out.splitlines(): if line.endswith(" path %s" % subvol): - subvolid = line.split(" ")[1] + subvolid = line.split()[1] if not subvolid.isdigit(): raise MountError("Invalid subvolume id: %s" % subvolid) subvolid = int(subvolid) @@ -385,20 +365,23 @@ class PartitionedMount(Mount): def __create_subvolume_metadata(self, p, pdisk): if len(self.subvolumes) == 0: return + argv = [ self.btrfscmd, "subvolume", "list", pdisk.mountdir ] - p1 = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - (out,err) = p1.communicate() + rc, out = runner.runtool(argv) msger.debug(out) - if p1.returncode != 0: - raise MountError("Failed to get subvolume id from %s', return code: %d." % (pdisk.mountdir, p1.returncode)) - subvolid_items = out.split("\n") + + if rc != 0: + raise MountError("Failed to get subvolume id from %s', return code: %d." % (pdisk.mountdir, rc)) + + subvolid_items = out.splitlines() subvolume_metadata = "" for subvol in self.subvolumes: for line in subvolid_items: if line.endswith(" path %s" % subvol["subvol"]): - subvolid = line.split(" ")[1] + subvolid = line.split()[1] if not subvolid.isdigit(): raise MountError("Invalid subvolume id: %s" % subvolid) + subvolid = int(subvolid) opts = subvol["fsopts"].split(",") for opt in opts: @@ -407,6 +390,7 @@ class PartitionedMount(Mount): break fsopts = ",".join(opts) subvolume_metadata += "%d\t%s\t%s\t%s\n" % (subvolid, subvol["subvol"], subvol['mountpoint'], fsopts) + if subvolume_metadata: fd = open("%s/.subvolume_metadata" % pdisk.mountdir, "w") fd.write(subvolume_metadata) @@ -416,10 +400,12 @@ class PartitionedMount(Mount): subvolume_metadata_file = "%s/.subvolume_metadata" % pdisk.mountdir if not os.path.exists(subvolume_metadata_file): return + fd = open(subvolume_metadata_file, "r") content = fd.read() fd.close() - for line in content.split("\n"): + + for line in content.splitlines(): items = line.split("\t") if items and len(items) == 4: self.subvolumes.append({'size': 0, # In sectors @@ -436,13 +422,13 @@ class PartitionedMount(Mount): def __create_subvolumes(self, p, pdisk): """ Create all the subvolumes """ + for subvol in self.subvolumes: argv = [ self.btrfscmd, "subvolume", "create", pdisk.mountdir + "/" + subvol["subvol"]] - p1 = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - (out,err) = p1.communicate() - msger.debug(out) - if p1.returncode != 0: - raise MountError("Failed to create subvolume '%s', return code: %d." % (subvol["subvol"], p1.returncode)) + + rc = runner.show(argv) + if rc != 0: + raise MountError("Failed to create subvolume '%s', return code: %d." % (subvol["subvol"], rc)) """ Set default subvolume, subvolume for "/" is default """ subvol = None @@ -450,17 +436,15 @@ class PartitionedMount(Mount): if subvolume["mountpoint"] == "/" and p["disk"] == subvolume["disk"]: subvol = subvolume break + if subvol: """ Get default subvolume id """ subvolid = self. __get_subvolume_id(pdisk.mountdir, subvol["subvol"]) """ Set default subvolume """ if subvolid != -1: - argv = [ self.btrfscmd, "subvolume", "set-default", "%d" % subvolid, pdisk.mountdir] - p1 = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - (out,err) = p1.communicate() - msger.debug(out) - if p1.returncode != 0: - raise MountError("Failed to set default subvolume id: %d', return code: %d." % (subvolid, p1.returncode)) + rc = runner.show([ self.btrfscmd, "subvolume", "set-default", "%d" % subvolid, pdisk.mountdir]) + if rc != 0: + raise MountError("Failed to set default subvolume id: %d', return code: %d." % (subvolid, rc)) self.__create_subvolume_metadata(p, pdisk) @@ -485,12 +469,14 @@ class PartitionedMount(Mount): return """ Remount to make default subvolume mounted """ - rc = subprocess.call([self.umountcmd, pdisk.mountdir]) + rc = runner.show([self.umountcmd, pdisk.mountdir]) if rc != 0: raise MountError("Failed to umount %s" % pdisk.mountdir) - rc = subprocess.call([self.mountcmd, "-o", pdisk.fsopts, pdisk.disk.device, pdisk.mountdir]) + + rc = runner.show([self.mountcmd, "-o", pdisk.fsopts, pdisk.disk.device, pdisk.mountdir]) if rc != 0: raise MountError("Failed to umount %s" % pdisk.mountdir) + for subvol in self.subvolumes: if subvol["mountpoint"] == "/": continue @@ -510,7 +496,7 @@ class PartitionedMount(Mount): subvol['fsopts'] = fsopts mountpoint = self.mountdir + subvol['mountpoint'] makedirs(mountpoint) - rc = subprocess.call([self.mountcmd, "-o", fsopts, pdisk.disk.device, mountpoint]) + rc = runner.show([self.mountcmd, "-o", fsopts, pdisk.disk.device, mountpoint]) if rc != 0: raise MountError("Failed to mount subvolume %s to %s" % (subvol["subvol"], mountpoint)) subvol["mounted"] = True @@ -523,24 +509,26 @@ class PartitionedMount(Mount): if not subvol["mounted"]: continue mountpoint = self.mountdir + subvol['mountpoint'] - rc = subprocess.call([self.umountcmd, mountpoint]) + rc = runner.show([self.umountcmd, mountpoint]) if rc != 0: raise MountError("Failed to unmount subvolume %s from %s" % (subvol["subvol"], mountpoint)) subvol["mounted"] = False def __create_subvolume_snapshots(self, p, pdisk): + import time + if self.snapshot_created: return """ Remount with subvolid=0 """ - rc = subprocess.call([self.umountcmd, pdisk.mountdir]) + rc = runner.show([self.umountcmd, pdisk.mountdir]) if rc != 0: raise MountError("Failed to umount %s" % pdisk.mountdir) if pdisk.fsopts: mountopts = pdisk.fsopts + ",subvolid=0" else: mountopts = "subvolid=0" - rc = subprocess.call([self.mountcmd, "-o", mountopts, pdisk.disk.device, pdisk.mountdir]) + rc = runner.show([self.mountcmd, "-o", mountopts, pdisk.disk.device, pdisk.mountdir]) if rc != 0: raise MountError("Failed to umount %s" % pdisk.mountdir) @@ -549,12 +537,10 @@ class PartitionedMount(Mount): for subvol in self.subvolumes: subvolpath = pdisk.mountdir + "/" + subvol["subvol"] snapshotpath = subvolpath + "_%s-1" % snapshotts - argv = [ self.btrfscmd, "subvolume", "snapshot", subvolpath, snapshotpath ] - p1 = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - (out,err) = p1.communicate() - msger.debug(out) - if p1.returncode != 0: - raise MountError("Failed to create subvolume snapshot '%s' for '%s', return code: %d." % (snapshotpath, subvolpath, p1.returncode)) + rc = runner.show([ self.btrfscmd, "subvolume", "snapshot", subvolpath, snapshotpath ]) + if rc != 0: + raise MountError("Failed to create subvolume snapshot '%s' for '%s', return code: %d." % (snapshotpath, subvolpath, rc)) + self.snapshot_created = True def mount(self): @@ -574,7 +560,7 @@ class PartitionedMount(Mount): break if mp == 'swap': - subprocess.call([self.mkswap, p['device']]) + runner.show([self.mkswap, p['device']]) continue rmmountdir = False diff --git a/plugins/imager/fs_plugin.py b/plugins/imager/fs_plugin.py index ca8b8ba..f6c7d3f 100644 --- a/plugins/imager/fs_plugin.py +++ b/plugins/imager/fs_plugin.py @@ -99,8 +99,6 @@ class FsPlugin(ImagerPlugin): def do_chroot(self, target):#chroot.py parse opts&args try: chroot.chroot(target, None, "/bin/env HOME=/root /bin/bash") - except: - msger.warning("Failed to chroot to %s." % target) finally: chroot.cleanup_after_chroot("dir", None, None, None) return 1 diff --git a/plugins/imager/raw_plugin.py b/plugins/imager/raw_plugin.py index c2b1345..914b72a 100644 --- a/plugins/imager/raw_plugin.py +++ b/plugins/imager/raw_plugin.py @@ -84,8 +84,6 @@ class RawPlugin(ImagerPlugin): @classmethod def do_chroot(cls, target): - import subprocess - img = target imgsize = misc.get_file_size(img) * 1024L * 1024L partedcmd = fs_related.find_binary_path("parted") @@ -97,12 +95,7 @@ class RawPlugin(ImagerPlugin): # Check the partitions from raw disk. root_mounted = False partition_mounts = 0 - for line in subprocess.Popen([partedcmd,"-s",img,"unit","B","print"], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\ - .communicate()[0]\ - .strip()\ - .splitlines(): - + for line in runner.outs([partedcmd,"-s",img,"unit","B","print"]).splitlines(): line = line.strip() # Lines that start with number are the partitions,