2 # partitionedfs.py: partitioned files system class, extends fs.py
4 # Copyright 2007-2008, Red Hat Inc.
5 # Copyright 2008, Daniel P. Berrange
6 # Copyright 2008, David P. Huff
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; version 2 of the License.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Library General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
27 from mic.utils.errors import *
28 from mic.utils.fs_related import *
31 class PartitionedMount(Mount):
32 def __init__(self, disks, mountdir, skipformat = False):
33 Mount.__init__(self, mountdir)
35 for name in disks.keys():
36 self.disks[name] = { 'disk': disks[name], # Disk object
37 'mapped': False, # True if kpartx mapping exists
38 'numpart': 0, # Number of allocate partitions
39 'partitions': [], # indexes to self.partitions
40 # Partitions with part num higher than 3 will
41 # be put inside extended partition.
42 'extended': 0, # Size of extended partition
43 # Sector 0 is used by the MBR and can't be used
44 # as the start, so setting offset to 1.
45 'offset': 1 } # Offset of next partition (in sectors)
51 self.unmountOrder = []
52 self.parted=find_binary_path("parted")
53 self.kpartx=find_binary_path("kpartx")
54 self.mkswap=find_binary_path("mkswap")
56 self.mountcmd=find_binary_path("mount")
57 self.umountcmd=find_binary_path("umount")
58 self.skipformat = skipformat
59 self.snapshot_created = self.skipformat
60 # Size of a sector used in calculations
61 self.sector_size = 512
63 def add_partition(self, size, disk, mountpoint, fstype = None, fsopts = None, boot = False):
64 # Converting M to s for parted
65 size = size * 1024 * 1024 / self.sector_size
67 """ We need to handle subvolumes for btrfs """
68 if fstype == "btrfs" and fsopts and fsopts.find("subvol=") != -1:
69 self.btrfscmd=find_binary_path("btrfs")
71 opts = fsopts.split(",")
73 if opt.find("subvol=") != -1:
74 subvol = opt.replace("subvol=", "").strip()
77 raise MountError("No subvolume: %s" % fsopts)
78 self.subvolumes.append({'size': size, # In sectors
79 'mountpoint': mountpoint, # Mount relative to chroot
80 'fstype': fstype, # Filesystem type
81 'fsopts': fsopts, # Filesystem mount options
82 'disk': disk, # physical disk name holding partition
83 'device': None, # kpartx device node for partition
84 'mount': None, # Mount object
85 'subvol': subvol, # Subvolume name
86 'boot': boot, # Bootable flag
87 'mounted': False # Mount flag
90 """ We still need partition for "/" or non-subvolume """
91 if mountpoint == "/" or not fsopts or fsopts.find("subvol=") == -1:
92 """ Don't need subvolume for "/" because it will be set as default subvolume """
93 if fsopts and fsopts.find("subvol=") != -1:
94 opts = fsopts.split(",")
96 if opt.strip().startswith("subvol="):
99 fsopts = ",".join(opts)
100 self.partitions.append({'size': size, # In sectors
101 'mountpoint': mountpoint, # Mount relative to chroot
102 'fstype': fstype, # Filesystem type
103 'fsopts': fsopts, # Filesystem mount options
104 'disk': disk, # physical disk name holding partition
105 'device': None, # kpartx device node for partition
106 'mount': None, # Mount object
107 'num': None, # Partition number
108 'boot': boot}) # Bootable flag
110 def __create_part_to_image(self,device, parttype, fstype, start, size):
111 # Start is included to the size so we need to substract one from the end.
113 msger.debug("Added '%s' part at %d of size %d" % (parttype,start,end))
114 part_cmd = [self.parted, "-s", device, "unit", "s", "mkpart", parttype]
116 part_cmd.extend([fstype])
117 part_cmd.extend(["%d" % start, "%d" % end])
119 msger.debug(part_cmd)
120 p1 = subprocess.Popen(part_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
121 out = p1.communicate()[0].strip()
123 msger.debug('"parted" output: %s' % out)
126 def __format_disks(self):
127 msger.debug("Assigning partitions to disks")
129 mbr_sector_skipped = False
131 for n in range(len(self.partitions)):
132 p = self.partitions[n]
134 if not self.disks.has_key(p['disk']):
135 raise MountError("No disk %s for partition %s" % (p['disk'], p['mountpoint']))
137 if not mbr_sector_skipped:
138 # This hack is used to remove one sector from the first partition,
139 # that is the used to the MBR.
141 mbr_sector_skipped = True
143 d = self.disks[p['disk']]
146 # Increase allocation of extended partition to hold this partition
147 d['extended'] += p['size']
148 p['type'] = 'logical'
149 p['num'] = d['numpart'] + 1
151 p['type'] = 'primary'
152 p['num'] = d['numpart']
154 p['start'] = d['offset']
155 d['offset'] += p['size']
156 d['partitions'].append(n)
157 msger.debug("Assigned %s to %s%d at %d at size %d" % (p['mountpoint'], p['disk'], p['num'], p['start'], p['size']))
160 msger.debug("Skipping disk format, because skipformat flag is set.")
163 for dev in self.disks.keys():
165 msger.debug("Initializing partition table for %s" % (d['disk'].device))
166 p1 = subprocess.Popen([self.parted, "-s", d['disk'].device, "mklabel", "msdos"],
167 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
168 out = p1.communicate()[0].strip()
170 msger.debug('"parted" output: %s' % out)
172 if p1.returncode != 0:
173 # NOTE: We don't throw exception when return code is not 0, because
174 # parted always fails to reload part table with loop devices.
175 # This prevents us from distinguishing real errors based on return code.
176 msger.debug("WARNING: parted returned '%s' instead of 0 when creating partition-table for disk '%s'." % (p1.returncode,d['disk'].device))
178 msger.debug("Creating partitions")
180 for p in self.partitions:
181 d = self.disks[p['disk']]
183 self.__create_part_to_image(d['disk'].device,"extended",None,p['start'],d['extended'])
185 if p['fstype'] == "swap":
186 parted_fs_type = "linux-swap"
187 elif p['fstype'] == "vfat":
188 parted_fs_type = "fat32"
189 elif p['fstype'] == "msdos":
190 parted_fs_type = "fat16"
192 # Type for ext2/ext3/ext4/btrfs
193 parted_fs_type = "ext2"
195 # Boot ROM of OMAP boards require vfat boot partition to have an
196 # even number of sectors.
197 if p['mountpoint'] == "/boot" and p['fstype'] in ["vfat","msdos"] and p['size'] % 2:
198 msger.debug("Substracting one sector from '%s' partition to get even number of sectors for the partition." % (p['mountpoint']))
201 ret = self.__create_part_to_image(d['disk'].device,p['type'],
202 parted_fs_type, p['start'],
206 # NOTE: We don't throw exception when return code is not 0, because
207 # parted always fails to reload part table with loop devices.
208 # This prevents us from distinguishing real errors based on return code.
209 msger.debug("WARNING: parted returned '%s' instead of 0 when creating partition '%s' for disk '%s'." % (p1.returncode,p['mountpoint'],d['disk'].device))
212 msger.debug("Setting boot flag for partition '%s' on disk '%s'." % (p['num'],d['disk'].device))
213 boot_cmd = [self.parted, "-s", d['disk'].device, "set", "%d" % p['num'], "boot", "on"]
214 msger.debug(boot_cmd)
215 p1 = subprocess.Popen(boot_cmd,
216 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
217 (out,err) = p1.communicate()
220 if p1.returncode != 0:
221 # NOTE: We don't throw exception when return code is not 0, because
222 # parted always fails to reload part table with loop devices.
223 # This prevents us from distinguishing real errors based on return code.
224 msger.debug("WARNING: parted returned '%s' instead of 0 when adding boot flag for partition '%s' disk '%s'." % (p1.returncode,p['num'],d['disk'].device))
226 def __map_partitions(self):
227 """Load it if dm_snapshot isn't loaded"""
228 load_module("dm_snapshot")
230 dev_null = os.open("/dev/null", os.O_WRONLY)
231 for dev in self.disks.keys():
236 msger.debug("Running kpartx on %s" % d['disk'].device )
237 kpartx = subprocess.Popen([self.kpartx, "-l", "-v", d['disk'].device],
238 stdout=subprocess.PIPE, stderr=dev_null)
240 kpartxOutput = kpartx.communicate()[0].strip().split("\n")
242 if kpartx.returncode:
244 raise MountError("Failed to query partition mapping for '%s'" %
247 # Strip trailing blank and mask verbose output
249 while i < len(kpartxOutput) and kpartxOutput[i][0:4] != "loop":
251 kpartxOutput = kpartxOutput[i:]
253 # Quick sanity check that the number of partitions matches
254 # our expectation. If it doesn't, someone broke the code
256 if len(kpartxOutput) != d['numpart']:
258 raise MountError("Unexpected number of partitions from kpartx: %d != %d" %
259 (len(kpartxOutput), d['numpart']))
261 for i in range(len(kpartxOutput)):
262 line = kpartxOutput[i]
263 newdev = line.split()[0]
264 mapperdev = "/dev/mapper/" + newdev
265 loopdev = d['disk'].device + newdev[-1]
267 msger.debug("Dev %s: %s -> %s" % (newdev, loopdev, mapperdev))
268 pnum = d['partitions'][i]
269 self.partitions[pnum]['device'] = loopdev
271 # grub's install wants partitions to be named
272 # to match their parent device + partition num
273 # kpartx doesn't work like this, so we add compat
274 # symlinks to point to /dev/mapper
275 if os.path.lexists(loopdev):
277 os.symlink(mapperdev, loopdev)
279 msger.debug("Adding partx mapping for %s" % d['disk'].device)
280 p1 = subprocess.Popen([self.kpartx, "-v", "-a", d['disk'].device],
281 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
283 (out,err) = p1.communicate()
286 if p1.returncode != 0:
287 # Make sure that the device maps are also removed on error case.
288 # The d['mapped'] isn't set to True if the kpartx fails so
289 # failed mapping will not be cleaned on cleanup either.
290 subprocess.call([self.kpartx, "-d", d['disk'].device],
291 stdout=dev_null, stderr=dev_null)
293 raise MountError("Failed to map partitions for '%s'" %
299 def __unmap_partitions(self):
300 dev_null = os.open("/dev/null", os.O_WRONLY)
301 for dev in self.disks.keys():
306 msger.debug("Removing compat symlinks")
307 for pnum in d['partitions']:
308 if self.partitions[pnum]['device'] != None:
309 os.unlink(self.partitions[pnum]['device'])
310 self.partitions[pnum]['device'] = None
312 msger.debug("Unmapping %s" % d['disk'].device)
313 rc = subprocess.call([self.kpartx, "-d", d['disk'].device],
314 stdout=dev_null, stderr=dev_null)
317 raise MountError("Failed to unmap partitions for '%s'" %
324 def __calculate_mountorder(self):
325 msger.debug("Calculating mount order")
326 for p in self.partitions:
327 self.mountOrder.append(p['mountpoint'])
328 self.unmountOrder.append(p['mountpoint'])
330 self.mountOrder.sort()
331 self.unmountOrder.sort()
332 self.unmountOrder.reverse()
336 self.__unmap_partitions()
337 for dev in self.disks.keys():
345 self.__unmount_subvolumes()
346 for mp in self.unmountOrder:
350 for p1 in self.partitions:
351 if p1['mountpoint'] == mp:
355 if p['mount'] != None:
357 """ Create subvolume snapshot here """
358 if p['fstype'] == "btrfs" and p['mountpoint'] == "/" and not self.snapshot_created:
359 self.__create_subvolume_snapshots(p, p["mount"])
365 """ Only for btrfs """
366 def __get_subvolume_id(self, rootpath, subvol):
367 if not self.btrfscmd:
368 self.btrfscmd=find_binary_path("btrfs")
369 argv = [ self.btrfscmd, "subvolume", "list", rootpath ]
370 p1 = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
371 (out,err) = p1.communicate()
373 if p1.returncode != 0:
374 raise MountError("Failed to get subvolume id from %s', return code: %d." % (rootpath, p1.returncode))
376 for line in out.split("\n"):
377 if line.endswith(" path %s" % subvol):
378 subvolid = line.split(" ")[1]
379 if not subvolid.isdigit():
380 raise MountError("Invalid subvolume id: %s" % subvolid)
381 subvolid = int(subvolid)
385 def __create_subvolume_metadata(self, p, pdisk):
386 if len(self.subvolumes) == 0:
388 argv = [ self.btrfscmd, "subvolume", "list", pdisk.mountdir ]
389 p1 = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
390 (out,err) = p1.communicate()
392 if p1.returncode != 0:
393 raise MountError("Failed to get subvolume id from %s', return code: %d." % (pdisk.mountdir, p1.returncode))
394 subvolid_items = out.split("\n")
395 subvolume_metadata = ""
396 for subvol in self.subvolumes:
397 for line in subvolid_items:
398 if line.endswith(" path %s" % subvol["subvol"]):
399 subvolid = line.split(" ")[1]
400 if not subvolid.isdigit():
401 raise MountError("Invalid subvolume id: %s" % subvolid)
402 subvolid = int(subvolid)
403 opts = subvol["fsopts"].split(",")
405 if opt.strip().startswith("subvol="):
408 fsopts = ",".join(opts)
409 subvolume_metadata += "%d\t%s\t%s\t%s\n" % (subvolid, subvol["subvol"], subvol['mountpoint'], fsopts)
410 if subvolume_metadata:
411 fd = open("%s/.subvolume_metadata" % pdisk.mountdir, "w")
412 fd.write(subvolume_metadata)
415 def __get_subvolume_metadata(self, p, pdisk):
416 subvolume_metadata_file = "%s/.subvolume_metadata" % pdisk.mountdir
417 if not os.path.exists(subvolume_metadata_file):
419 fd = open(subvolume_metadata_file, "r")
422 for line in content.split("\n"):
423 items = line.split("\t")
424 if items and len(items) == 4:
425 self.subvolumes.append({'size': 0, # In sectors
426 'mountpoint': items[2], # Mount relative to chroot
427 'fstype': "btrfs", # Filesystem type
428 'fsopts': items[3] + ",subvol=%s" % items[1], # Filesystem mount options
429 'disk': p['disk'], # physical disk name holding partition
430 'device': None, # kpartx device node for partition
431 'mount': None, # Mount object
432 'subvol': items[1], # Subvolume name
433 'boot': False, # Bootable flag
434 'mounted': False # Mount flag
437 def __create_subvolumes(self, p, pdisk):
438 """ Create all the subvolumes """
439 for subvol in self.subvolumes:
440 argv = [ self.btrfscmd, "subvolume", "create", pdisk.mountdir + "/" + subvol["subvol"]]
441 p1 = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
442 (out,err) = p1.communicate()
444 if p1.returncode != 0:
445 raise MountError("Failed to create subvolume '%s', return code: %d." % (subvol["subvol"], p1.returncode))
447 """ Set default subvolume, subvolume for "/" is default """
449 for subvolume in self.subvolumes:
450 if subvolume["mountpoint"] == "/" and p["disk"] == subvolume["disk"]:
454 """ Get default subvolume id """
455 subvolid = self. __get_subvolume_id(pdisk.mountdir, subvol["subvol"])
456 """ Set default subvolume """
458 argv = [ self.btrfscmd, "subvolume", "set-default", "%d" % subvolid, pdisk.mountdir]
459 p1 = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
460 (out,err) = p1.communicate()
462 if p1.returncode != 0:
463 raise MountError("Failed to set default subvolume id: %d', return code: %d." % (subvolid, p1.returncode))
465 self.__create_subvolume_metadata(p, pdisk)
467 def __mount_subvolumes(self, p, pdisk):
469 """ Get subvolume info """
470 self.__get_subvolume_metadata(p, pdisk)
471 """ Set default mount options """
472 if len(self.subvolumes) != 0:
473 for subvol in self.subvolumes:
474 if subvol["mountpoint"] == p["mountpoint"] == "/":
475 opts = subvol["fsopts"].split(",")
477 if opt.strip().startswith("subvol="):
480 pdisk.fsopts = ",".join(opts)
483 if len(self.subvolumes) == 0:
484 """ Return directly if no subvolumes """
487 """ Remount to make default subvolume mounted """
488 rc = subprocess.call([self.umountcmd, pdisk.mountdir])
490 raise MountError("Failed to umount %s" % pdisk.mountdir)
491 rc = subprocess.call([self.mountcmd, "-o", pdisk.fsopts, pdisk.disk.device, pdisk.mountdir])
493 raise MountError("Failed to umount %s" % pdisk.mountdir)
494 for subvol in self.subvolumes:
495 if subvol["mountpoint"] == "/":
497 subvolid = self. __get_subvolume_id(pdisk.mountdir, subvol["subvol"])
499 msger.debug("WARNING: invalid subvolume %s" % subvol["subvol"])
501 """ Replace subvolume name with subvolume ID """
502 opts = subvol["fsopts"].split(",")
504 if opt.strip().startswith("subvol="):
507 #opts.append("subvolid=%d" % subvolid)
508 opts.extend(["subvolrootid=0", "subvol=%s" % subvol["subvol"]])
509 fsopts = ",".join(opts)
510 subvol['fsopts'] = fsopts
511 mountpoint = self.mountdir + subvol['mountpoint']
513 rc = subprocess.call([self.mountcmd, "-o", fsopts, pdisk.disk.device, mountpoint])
515 raise MountError("Failed to mount subvolume %s to %s" % (subvol["subvol"], mountpoint))
516 subvol["mounted"] = True
518 def __unmount_subvolumes(self):
519 """ It may be called multiple times, so we need to chekc if it is still mounted. """
520 for subvol in self.subvolumes:
521 if subvol["mountpoint"] == "/":
523 if not subvol["mounted"]:
525 mountpoint = self.mountdir + subvol['mountpoint']
526 rc = subprocess.call([self.umountcmd, mountpoint])
528 raise MountError("Failed to unmount subvolume %s from %s" % (subvol["subvol"], mountpoint))
529 subvol["mounted"] = False
531 def __create_subvolume_snapshots(self, p, pdisk):
532 if self.snapshot_created:
535 """ Remount with subvolid=0 """
536 rc = subprocess.call([self.umountcmd, pdisk.mountdir])
538 raise MountError("Failed to umount %s" % pdisk.mountdir)
540 mountopts = pdisk.fsopts + ",subvolid=0"
542 mountopts = "subvolid=0"
543 rc = subprocess.call([self.mountcmd, "-o", mountopts, pdisk.disk.device, pdisk.mountdir])
545 raise MountError("Failed to umount %s" % pdisk.mountdir)
547 """ Create all the subvolume snapshots """
548 snapshotts = time.strftime("%Y%m%d-%H%M")
549 for subvol in self.subvolumes:
550 subvolpath = pdisk.mountdir + "/" + subvol["subvol"]
551 snapshotpath = subvolpath + "_%s-1" % snapshotts
552 argv = [ self.btrfscmd, "subvolume", "snapshot", subvolpath, snapshotpath ]
553 p1 = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
554 (out,err) = p1.communicate()
556 if p1.returncode != 0:
557 raise MountError("Failed to create subvolume snapshot '%s' for '%s', return code: %d." % (snapshotpath, subvolpath, p1.returncode))
558 self.snapshot_created = True
561 for dev in self.disks.keys():
565 self.__format_disks()
566 self.__map_partitions()
567 self.__calculate_mountorder()
569 for mp in self.mountOrder:
571 for p1 in self.partitions:
572 if p1['mountpoint'] == mp:
577 subprocess.call([self.mkswap, p['device']])
581 if p['mountpoint'] == "/":
583 if p['fstype'] == "vfat" or p['fstype'] == "msdos":
584 myDiskMount = VfatDiskMount
585 elif p['fstype'] in ("ext2", "ext3", "ext4"):
586 myDiskMount = ExtDiskMount
587 elif p['fstype'] == "btrfs":
588 myDiskMount = BtrfsDiskMount
590 raise MountError("Fail to support file system " + p['fstype'])
592 if p['fstype'] == "btrfs" and not p['fsopts']:
593 p['fsopts'] = "subvolid=0"
595 pdisk = myDiskMount(RawDisk(p['size'] * self.sector_size, p['device']),
596 self.mountdir + p['mountpoint'],
602 fsopts = p['fsopts'])
603 pdisk.mount(pdisk.fsopts)
604 if p['fstype'] == "btrfs" and p['mountpoint'] == "/":
605 if not self.skipformat:
606 self.__create_subvolumes(p, pdisk)
607 self.__mount_subvolumes(p, pdisk)
610 def resparse(self, size = None):
611 # Can't re-sparse a disk image - too hard