3 # Copyright (c) 2007, Red Hat, Inc.
4 # Copyright (c) 2009, 2010, 2011 Intel, Inc.
6 # This program is free software; you can redistribute it and/or modify it
7 # under the terms of the GNU General Public License as published by the Free
8 # Software Foundation; version 2 of the License
10 # This program is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 # You should have received a copy of the GNU General Public License along
16 # with this program; if not, write to the Free Software Foundation, Inc., 59
17 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 from __future__ import with_statement
30 from mic.utils import runner
31 from mic.utils.errors import *
34 def find_binary_inchroot(binary, chroot):
42 bin_path = "%s/%s" % (path, binary)
43 if os.path.exists("%s/%s" % (chroot, bin_path)):
47 def find_binary_path(binary):
48 if os.environ.has_key("PATH"):
49 paths = os.environ["PATH"].split(":")
52 if os.environ.has_key("HOME"):
53 paths += [os.environ["HOME"] + "/bin"]
54 paths += ["/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"]
57 bin_path = "%s/%s" % (path, binary)
58 if os.path.exists(bin_path):
60 raise CreatorError("Command '%s' is not available." % binary)
62 def makedirs(dirname):
63 """A version of os.makedirs() that doesn't throw an
64 exception if the leaf directory already exists.
69 if err.errno != errno.EEXIST:
72 def mksquashfs(in_img, out_img):
73 fullpathmksquashfs = find_binary_path("mksquashfs")
74 args = [fullpathmksquashfs, in_img, out_img]
76 if not sys.stdout.isatty():
77 args.append("-no-progress")
79 ret = runner.show(args)
81 raise SquashfsError("'%s' exited with error (%d)" % (' '.join(args), ret))
83 def resize2fs(fs, size):
84 resize2fs = find_binary_path("resize2fs")
86 # it means to minimalize it
87 return runner.show([resize2fs, '-M', fs])
89 return runner.show([resize2fs, fs, "%sK" % (size / 1024,)])
91 class BindChrootMount:
92 """Represents a bind mount of a directory into a chroot."""
93 def __init__(self, src, chroot, dest = None, option = None):
94 self.root = os.path.abspath(os.path.expanduser(chroot))
97 self.orig_src = self.src = src
98 if os.path.islink(src):
99 self.src = os.readlink(src)
100 if not self.src.startswith('/'):
101 self.src = os.path.abspath(os.path.join(os.path.dirname(src),
106 self.dest = os.path.join(self.root, dest.lstrip('/'))
109 self.mountcmd = find_binary_path("mount")
110 self.umountcmd = find_binary_path("umount")
113 with open('/proc/mounts') as f:
115 if line.split()[1] == os.path.abspath(self.dest):
121 if self.mounted or self.ismounted():
125 rc = runner.show([self.mountcmd, "--bind", self.src, self.dest])
127 raise MountError("Bind-mounting '%s' to '%s' failed" %
128 (self.src, self.dest))
130 rc = runner.show([self.mountcmd, "--bind", "-o", "remount,%s" % self.option, self.dest])
132 raise MountError("Bind-remounting '%s' failed" % self.dest)
135 if os.path.islink(self.orig_src):
136 dest = os.path.join(self.root, self.orig_src.lstrip('/'))
137 if not os.path.exists(dest):
138 os.symlink(self.src, dest)
141 if self.mounted or self.ismounted():
142 runner.show([self.umountcmd, "-l", self.dest])
146 """LoopbackMount compatibility layer for old API"""
147 def __init__(self, lofile, mountdir, fstype = None):
148 self.diskmount = DiskMount(LoopbackDisk(lofile,size = 0),mountdir,fstype,rmmountdir = True)
150 self.losetupcmd = find_binary_path("losetup")
153 self.diskmount.cleanup()
156 self.diskmount.unmount()
160 runner.show([self.losetupcmd, "-d", self.loopdev])
168 self.loopdev = get_loop_device(self.losetupcmd, self.lofile)
172 self.diskmount.mount()
174 class SparseLoopbackMount(LoopbackMount):
175 """SparseLoopbackMount compatibility layer for old API"""
176 def __init__(self, lofile, mountdir, size, fstype = None):
177 self.diskmount = DiskMount(SparseLoopbackDisk(lofile,size),mountdir,fstype,rmmountdir = True)
179 def expand(self, create = False, size = None):
180 self.diskmount.disk.expand(create, size)
182 def truncate(self, size = None):
183 self.diskmount.disk.truncate(size)
186 self.diskmount.disk.create()
188 class SparseExtLoopbackMount(SparseLoopbackMount):
189 """SparseExtLoopbackMount compatibility layer for old API"""
190 def __init__(self, lofile, mountdir, size, fstype, blocksize, fslabel):
191 self.diskmount = ExtDiskMount(SparseLoopbackDisk(lofile,size), mountdir, fstype, blocksize, fslabel, rmmountdir = True)
194 def __format_filesystem(self):
195 self.diskmount.__format_filesystem()
198 self.diskmount.disk.create()
200 def resize(self, size = None):
201 return self.diskmount.__resize_filesystem(size)
204 self.diskmount.mount()
207 self.extdiskmount.__fsck()
209 def __get_size_from_filesystem(self):
210 return self.diskmount.__get_size_from_filesystem()
212 def __resize_to_minimal(self):
213 return self.diskmount.__resize_to_minimal()
215 def resparse(self, size = None):
216 return self.diskmount.resparse(size)
219 """Generic base object for a disk
221 The 'create' method must make the disk visible as a block device - eg
222 by calling losetup. For RawDisk, this is obviously a no-op. The 'cleanup'
223 method must undo the 'create' operation.
225 def __init__(self, size, device = None):
226 self._device = device
235 def get_device(self):
237 def set_device(self, path):
239 device = property(get_device, set_device)
243 size = property(get_size)
247 """A Disk backed by a block device.
248 Note that create() is a no-op.
250 def __init__(self, size, device):
251 Disk.__init__(self, size, device)
259 class LoopbackDisk(Disk):
260 """A Disk backed by a file via the loop module."""
261 def __init__(self, lofile, size):
262 Disk.__init__(self, size)
264 self.losetupcmd = find_binary_path("losetup")
270 return os.path.exists(self.lofile)
273 if self.device is not None:
276 self.device = get_loop_device(self.losetupcmd, self.lofile)
279 if self.device is None:
281 msger.debug("Losetup remove %s" % self.device)
282 rc = runner.show([self.losetupcmd, "-d", self.device])
285 class SparseLoopbackDisk(LoopbackDisk):
286 """A Disk backed by a sparse file via the loop module."""
287 def __init__(self, lofile, size):
288 LoopbackDisk.__init__(self, lofile, size)
290 def expand(self, create = False, size = None):
294 if not os.path.exists(self.lofile):
295 makedirs(os.path.dirname(self.lofile))
300 msger.debug("Extending sparse file %s to %d" % (self.lofile, size))
302 fd = os.open(self.lofile, flags, 0644)
304 fd = os.open(self.lofile, flags)
309 os.ftruncate(fd, size)
311 # may be limited by 2G in 32bit env
312 os.ftruncate(fd, 2**31L)
316 def truncate(self, size = None):
320 msger.debug("Truncating sparse file %s to %d" % (self.lofile, size))
321 fd = os.open(self.lofile, os.O_WRONLY)
322 os.ftruncate(fd, size)
326 self.expand(create = True)
327 LoopbackDisk.create(self)
330 """A generic base class to deal with mounting things."""
331 def __init__(self, mountdir):
332 self.mountdir = mountdir
337 def mount(self, options = None):
343 class DiskMount(Mount):
344 """A Mount object that handles mounting of a Disk."""
345 def __init__(self, disk, mountdir, fstype = None, rmmountdir = True):
346 Mount.__init__(self, mountdir)
350 self.rmmountdir = rmmountdir
355 self.mkfscmd = find_binary_path("mkfs." + self.fstype)
358 self.mountcmd = find_binary_path("mount")
359 self.umountcmd = find_binary_path("umount")
367 msger.debug("Unmounting directory %s" % self.mountdir)
368 runner.quiet('sync') # sync the data on this mount point
369 rc = runner.show([self.umountcmd, "-l", self.mountdir])
373 raise MountError("Failed to umount %s" % self.mountdir)
374 if self.rmdir and not self.mounted:
376 os.rmdir(self.mountdir)
386 def mount(self, options = None):
390 if not os.path.isdir(self.mountdir):
391 msger.debug("Creating mount point %s" % self.mountdir)
392 os.makedirs(self.mountdir)
393 self.rmdir = self.rmmountdir
397 msger.debug("Mounting %s at %s" % (self.disk.device, self.mountdir))
399 args = [ self.mountcmd, "-o", options, self.disk.device, self.mountdir ]
401 args = [ self.mountcmd, self.disk.device, self.mountdir ]
403 args.extend(["-t", self.fstype])
405 rc = runner.show(args)
407 raise MountError("Failed to mount '%s' to '%s' with command '%s'. Retval: %s" %
408 (self.disk.device, self.mountdir, " ".join(args), rc))
412 class ExtDiskMount(DiskMount):
413 """A DiskMount object that is able to format/resize ext[23] filesystems."""
414 def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None):
415 DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
416 self.blocksize = blocksize
417 self.fslabel = fslabel.replace("/", "")
418 self.uuid = str(uuid.uuid4())
419 self.skipformat = skipformat
422 self.dumpe2fs = find_binary_path("dumpe2fs")
423 self.tune2fs = find_binary_path("tune2fs")
425 def __parse_field(self, output, field):
426 for line in output.split("\n"):
427 if line.startswith(field + ":"):
428 return line[len(field) + 1:].strip()
430 raise KeyError("Failed to find field '%s' in output" % field)
432 def __format_filesystem(self):
434 msger.debug("Skip filesystem format.")
437 msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
438 cmdlist = [self.mkfscmd, "-F", "-L", self.fslabel, "-m", "1", "-b",
439 str(self.blocksize), "-U", self.uuid]
441 cmdlist.extend(self.extopts.split())
442 cmdlist.extend([self.disk.device])
444 rc, errout = runner.runtool(cmdlist, catch=2)
446 raise MountError("Error creating %s filesystem on disk %s:\n%s" %
447 (self.fstype, self.disk.device, errout))
450 msger.debug("Tuning filesystem on %s" % self.disk.device)
451 runner.show([self.tune2fs, "-c0", "-i0", "-Odir_index", "-ouser_xattr,acl", self.disk.device])
453 def __resize_filesystem(self, size = None):
454 current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
457 size = self.disk.size
459 if size == current_size:
462 if size > current_size:
463 self.disk.expand(size)
467 resize2fs(self.disk.lofile, size)
472 if not self.disk.fixed() and self.disk.exists():
478 self.__resize_filesystem()
480 self.__format_filesystem()
482 def mount(self, options = None):
484 DiskMount.mount(self, options)
487 msger.info("Checking filesystem %s" % self.disk.lofile)
488 runner.quiet(["/sbin/e2fsck", "-f", "-y", self.disk.lofile])
490 def __get_size_from_filesystem(self):
491 return int(self.__parse_field(runner.outs([self.dumpe2fs, '-h', self.disk.lofile]),
492 "Block count")) * self.blocksize
494 def __resize_to_minimal(self):
498 # Use a binary search to find the minimal size
499 # we can resize the image to
502 top = self.__get_size_from_filesystem()
503 while top != (bot + 1):
504 t = bot + ((top - bot) / 2)
506 if not resize2fs(self.disk.lofile, t):
512 def resparse(self, size = None):
517 minsize = self.__resize_to_minimal()
518 self.disk.truncate(minsize)
520 self.__resize_filesystem(size)
523 class VfatDiskMount(DiskMount):
524 """A DiskMount object that is able to format vfat/msdos filesystems."""
525 def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None):
526 DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
527 self.blocksize = blocksize
528 self.fslabel = fslabel.replace("/", "")
529 rand1 = random.randint(0, 2**16 - 1)
530 rand2 = random.randint(0, 2**16 - 1)
531 self.uuid = "%04X-%04X" % (rand1, rand2)
532 self.skipformat = skipformat
534 self.fsckcmd = find_binary_path("fsck." + self.fstype)
536 def __format_filesystem(self):
538 msger.debug("Skip filesystem format.")
541 msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
542 rc = runner.show([self.mkfscmd, "-n", self.fslabel,
543 "-i", self.uuid.replace("-", ""), self.disk.device])
545 raise MountError("Error creating %s filesystem on disk %s" % (self.fstype,self.disk.device))
547 msger.verbose("Tuning filesystem on %s" % self.disk.device)
549 def __resize_filesystem(self, size = None):
550 current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
553 size = self.disk.size
555 if size == current_size:
558 if size > current_size:
559 self.disk.expand(size)
563 #resize2fs(self.disk.lofile, size)
568 if not self.disk.fixed() and self.disk.exists():
574 self.__resize_filesystem()
576 self.__format_filesystem()
578 def mount(self, options = None):
580 DiskMount.mount(self, options)
583 msger.debug("Checking filesystem %s" % self.disk.lofile)
584 runner.show([self.fsckcmd, "-y", self.disk.lofile])
586 def __get_size_from_filesystem(self):
587 return self.disk.size
589 def __resize_to_minimal(self):
593 # Use a binary search to find the minimal size
594 # we can resize the image to
597 top = self.__get_size_from_filesystem()
600 def resparse(self, size = None):
602 minsize = self.__resize_to_minimal()
603 self.disk.truncate(minsize)
604 self.__resize_filesystem(size)
607 class BtrfsDiskMount(DiskMount):
608 """A DiskMount object that is able to format/resize btrfs filesystems."""
609 def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None):
611 DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
612 self.blocksize = blocksize
613 self.fslabel = fslabel.replace("/", "")
615 self.skipformat = skipformat
617 self.blkidcmd = find_binary_path("blkid")
618 self.btrfsckcmd = find_binary_path("btrfsck")
620 def __check_btrfs(self):
622 """ Need to load btrfs module to mount it """
624 for line in open("/proc/filesystems").xreadlines():
625 if line.find("btrfs") > -1:
629 raise MountError("Your system can't mount btrfs filesystem, please make sure your kernel has btrfs support and the module btrfs.ko has been loaded.")
631 # disable selinux, selinux will block write
632 if os.path.exists("/usr/sbin/setenforce"):
633 runner.show(["/usr/sbin/setenforce", "0"])
635 def __parse_field(self, output, field):
636 for line in output.split(" "):
637 if line.startswith(field + "="):
638 return line[len(field) + 1:].strip().replace("\"", "")
640 raise KeyError("Failed to find field '%s' in output" % field)
642 def __format_filesystem(self):
644 msger.debug("Skip filesystem format.")
647 msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
648 rc = runner.show([self.mkfscmd, "-L", self.fslabel, self.disk.device])
650 raise MountError("Error creating %s filesystem on disk %s" % (self.fstype,self.disk.device))
652 self.uuid = self.__parse_field(runner.outs([self.blkidcmd, self.disk.device]), "UUID")
654 def __resize_filesystem(self, size = None):
655 current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
658 size = self.disk.size
660 if size == current_size:
663 if size > current_size:
664 self.disk.expand(size)
671 if not self.disk.fixed() and self.disk.exists():
677 self.__resize_filesystem()
679 self.__format_filesystem()
681 def mount(self, options = None):
683 DiskMount.mount(self, options)
686 msger.debug("Checking filesystem %s" % self.disk.lofile)
687 runner.quiet([self.btrfsckcmd, self.disk.lofile])
689 def __get_size_from_filesystem(self):
690 return self.disk.size
692 def __resize_to_minimal(self):
695 return self.__get_size_from_filesystem()
697 def resparse(self, size = None):
699 minsize = self.__resize_to_minimal()
700 self.disk.truncate(minsize)
701 self.__resize_filesystem(size)
704 class DeviceMapperSnapshot(object):
705 def __init__(self, imgloop, cowloop):
706 self.imgloop = imgloop
707 self.cowloop = cowloop
709 self.__created = False
711 self.dmsetupcmd = find_binary_path("dmsetup")
713 """Load dm_snapshot if it isn't loaded"""
714 load_module("dm_snapshot")
717 if self.__name is None:
719 return os.path.join("/dev/mapper", self.__name)
720 path = property(get_path)
726 self.imgloop.create()
727 self.cowloop.create()
729 self.__name = "imgcreate-%d-%d" % (os.getpid(),
730 random.randint(0, 2**16))
732 size = os.stat(self.imgloop.lofile)[stat.ST_SIZE]
734 table = "0 %d snapshot %s %s p 8" % (size / 512,
738 args = [self.dmsetupcmd, "create", self.__name, "--table", table]
739 if runner.show(args) != 0:
740 self.cowloop.cleanup()
741 self.imgloop.cleanup()
742 raise SnapshotError("Could not create snapshot device using: " + ' '.join(args))
744 self.__created = True
746 def remove(self, ignore_errors = False):
747 if not self.__created:
751 rc = runner.show([self.dmsetupcmd, "remove", self.__name])
752 if not ignore_errors and rc != 0:
753 raise SnapshotError("Could not remove snapshot device")
756 self.__created = False
758 self.cowloop.cleanup()
759 self.imgloop.cleanup()
761 def get_cow_used(self):
762 if not self.__created:
766 # dmsetup status on a snapshot returns e.g.
767 # "0 8388608 snapshot 416/1048576"
768 # or, more generally:
770 # where C is the number of 512 byte sectors in use
772 out = runner.outs([self.dmsetupcmd, "status", self.__name])
774 return int((out.split()[3]).split('/')[0]) * 512
776 raise SnapshotError("Failed to parse dmsetup status: " + out)
778 def create_image_minimizer(path, image, minimal_size):
780 Builds a copy-on-write image which can be used to
781 create a device-mapper snapshot of an image where
782 the image's filesystem is as small as possible
785 1) Create a sparse COW
786 2) Loopback mount the image and the COW
787 3) Create a device-mapper snapshot of the image
789 4) Resize the filesystem to the minimal size
790 5) Determine the amount of space used in the COW
791 6) Restroy the device-mapper snapshot
792 7) Truncate the COW, removing unused space
793 8) Create a squashfs of the COW
795 imgloop = LoopbackDisk(image, None) # Passing bogus size - doesn't matter
797 cowloop = SparseLoopbackDisk(os.path.join(os.path.dirname(path), "osmin"),
800 snapshot = DeviceMapperSnapshot(imgloop, cowloop)
805 resize2fs(snapshot.path, minimal_size)
807 cow_used = snapshot.get_cow_used()
809 snapshot.remove(ignore_errors = (not sys.exc_info()[0] is None))
811 cowloop.truncate(cow_used)
813 mksquashfs(cowloop.lofile, path)
815 os.unlink(cowloop.lofile)
817 def load_module(module):
819 for line in open('/proc/modules').xreadlines():
820 if line.startswith("%s " % module):
824 msger.info("Loading %s..." % module)
825 runner.quiet(['modprobe', module])
827 class LoopDevice(object):
828 def __init__(self, loopid=None):
832 self.kpartxcmd = find_binary_path("kpartx")
833 self.losetupcmd = find_binary_path("losetup")
835 def register(self, device):
840 def reg_atexit(self):
842 atexit.register(self.close)
844 def _genloopid(self):
846 if not glob.glob("/dev/loop[0-9]*"):
849 fint = lambda x: x[9:].isdigit() and int(x[9:]) or 0
850 maxid = 1 + max(filter(lambda x: x<100,
851 map(fint, glob.glob("/dev/loop[0-9]*"))))
852 if maxid < 10: maxid = 10
853 if maxid >= 100: raise
856 def _kpseek(self, device):
857 rc, out = runner.runtool([self.kpartxcmd, '-l', '-v', device])
859 raise MountError("Can't query dm snapshot on %s" % device)
860 for line in out.splitlines():
861 if line and line.startswith("loop"):
865 def _loseek(self, device):
867 rc, out = runner.runtool([self.losetupcmd, '-a'])
869 raise MountError("Failed to run 'losetup -a'")
870 for line in out.splitlines():
871 m = re.match("([^:]+): .*", line)
872 if m and m.group(1) == device:
879 self.loopid = self._genloopid()
880 self.device = "/dev/loop%d" % self.loopid
881 if os.path.exists(self.device):
882 if self._loseek(self.device):
883 raise MountError("Device busy: %s" % self.device)
888 mknod = find_binary_path('mknod')
889 rc = runner.show([mknod, '-m664', self.device, 'b', '7', str(self.loopid)])
891 raise MountError("Failed to create device %s" % self.device)
900 except MountError, e:
901 msger.error("%s" % e)
905 if self.device is None:
909 if self._kpseek(self.device):
911 for i in range(3, os.sysconf("SC_OPEN_MAX")):
916 runner.quiet([self.kpartxcmd, "-d", self.device])
917 if self._loseek(self.device):
918 runner.quiet([self.losetupcmd, "-d", self.device])
919 # FIXME: should sleep a while between two loseek
920 if self._loseek(self.device):
921 msger.warning("Can't cleanup loop device %s" % self.device)
923 os.unlink(self.device)
925 DEVICE_PIDFILE_DIR = "/var/tmp/mic/device"
926 DEVICE_LOCKFILE = "/var/lock/__mic_loopdev.lock"
928 def get_loop_device(losetupcmd, lofile):
929 global DEVICE_PIDFILE_DIR
930 global DEVICE_LOCKFILE
933 makedirs(os.path.dirname(DEVICE_LOCKFILE))
934 fp = open(DEVICE_LOCKFILE, 'w')
935 fcntl.flock(fp, fcntl.LOCK_EX)
938 devinst = LoopDevice()
940 # clean up left loop device first
943 # provide an avaible loop device
944 rc, out = runner.runtool([losetupcmd, "--find"])
946 loopdev = out.split()[0]
947 devinst.register(loopdev)
948 if not loopdev or not os.path.exists(loopdev):
950 loopdev = devinst.device
952 # setup a loop device for image file
953 rc = runner.show([losetupcmd, loopdev, lofile])
955 raise MountError("Failed to setup loop device for '%s'" % lofile)
959 # try to save device and pid
960 makedirs(DEVICE_PIDFILE_DIR)
961 pidfile = os.path.join(DEVICE_PIDFILE_DIR, os.path.basename(loopdev))
962 if os.path.exists(pidfile):
964 with open(pidfile, 'w') as wf:
965 wf.write(str(os.getpid()))
967 except MountError, err:
968 raise CreatorError("%s" % str(err))
973 fcntl.flock(fp, fcntl.LOCK_UN)
975 os.unlink(DEVICE_LOCKFILE)
981 def clean_loop_devices(piddir=DEVICE_PIDFILE_DIR):
982 if not os.path.exists(piddir) or not os.path.isdir(piddir):
985 for loopdev in os.listdir(piddir):
986 pidfile = os.path.join(piddir, loopdev)
988 with open(pidfile, 'r') as rf:
989 devpid = int(rf.read())
993 # if the process using this device is alive, skip it
994 if not devpid or os.path.exists(os.path.join('/proc', str(devpid))):
999 devinst = LoopDevice()
1000 devinst.register(os.path.join('/dev', loopdev))