create
------
-This command is used to create various images, including live CD, live USB,
-loop, raw.
+This command is used to create various images, including loop.
Usage:
| help(?) give detailed help on a specific sub-command
| fs create fs image, which is also chroot directory
| loop create loop image, including multi-partitions
- | raw create raw image, containing multi-partitions
Options:
--compress-image=COMPRESS_IMAGE compress all loop images with 'gz' or 'bz2' or 'lzo'
--compress-disk-image=COMPRESS_DISK_IMAGE same with --compress-image
-Options for raw image:
- --compress-image=COMPRESS_IMAGE compress all raw images with 'gz' or 'bz2' or 'lzo'
- --compress-disk-image=COMPRESS_DISK_IMAGE same with --compress-image
-
Examples:
| mic create loop tizen.ks
help(?) give detailed help on a specific sub-command
fs create fs image, which is also a chroot directory
loop create loop image, including multi-partitions
- raw create raw image, containing multi-partitions
- <ksfile>:
The kickstart file is a simple text file, containing a list of items about image partition, setup, Bootloader, packages to be installed, etc, each identified by a keyword.
-In Tizen, the released image will have a ks file along with image. For example, you can download the ks file from: http://download.tizen.org/releases/daily/trunk/ivi/latest/images/ivi-min...
+In Tizen, the released image will have a ks file along with image. For example, you can download the ks file from: http://download.tizen.org/releases/weekly/tizen/mobile/latest/images/...
- Options include:
+++ /dev/null
-#!/usr/bin/python -tt
-#
-# Copyright (c) 2011 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-import os
-import stat
-import shutil
-
-from mic import kickstart, msger
-from mic.utils import fs_related, runner, misc
-from mic.utils.partitionedfs import PartitionedMount
-from mic.utils.errors import CreatorError, MountError
-from mic.imager.baseimager import BaseImageCreator
-from mic.archive import packing, compressing
-
-class RawImageCreator(BaseImageCreator):
- """Installs a system into a file containing a partitioned disk image.
-
- ApplianceImageCreator is an advanced ImageCreator subclass; a sparse file
- is formatted with a partition table, each partition loopback mounted
- and the system installed into an virtual disk. The disk image can
- subsequently be booted in a virtual machine or accessed with kpartx
- """
- img_format = 'raw'
-
- def __init__(self, creatoropts=None, pkgmgr=None, compress_image=None, generate_bmap=None, fstab_entry="uuid"):
- """Initialize a ApplianceImageCreator instance.
-
- This method takes the same arguments as ImageCreator.__init__()
- """
- BaseImageCreator.__init__(self, creatoropts, pkgmgr)
-
- self.__instloop = None
- self.__imgdir = None
- self.__disks = {}
- self.__disk_format = "raw"
- self._disk_names = []
- self._ptable_format = self.ks.handler.bootloader.ptable
- self.vmem = 512
- self.vcpu = 1
- self.checksum = False
- self.use_uuid = fstab_entry == "uuid"
- self.appliance_version = None
- self.appliance_release = None
- self.compress_image = compress_image
- self.bmap_needed = generate_bmap
- self._need_extlinux = not kickstart.use_installerfw(self.ks, "bootloader")
- #self.getsource = False
- #self.listpkg = False
-
- self._dep_checks.extend(["sync", "kpartx", "parted"])
- if self._need_extlinux:
- self._dep_checks.extend(["extlinux"])
-
- def configure(self, repodata = None):
- import subprocess
- def chroot():
- os.chroot(self._instroot)
- os.chdir("/")
-
- if os.path.exists(self._instroot + "/usr/bin/Xorg"):
- subprocess.call(["/bin/chmod", "u+s", "/usr/bin/Xorg"],
- preexec_fn = chroot)
-
- BaseImageCreator.configure(self, repodata)
-
- def _get_fstab(self):
- s = ""
- for mp in self.__instloop.mount_order:
- p = None
- for p1 in self.__instloop.partitions:
- if p1['mountpoint'] == mp:
- p = p1
- break
-
- if self.use_uuid and p['uuid']:
- device = "UUID=%s" % p['uuid']
- else:
- device = "/dev/%s%-d" % (p['disk_name'], p['num'])
-
- s += "%(device)s %(mountpoint)s %(fstype)s %(fsopts)s 0 0\n" % {
- 'device': device,
- 'mountpoint': p['mountpoint'],
- 'fstype': p['fstype'],
- 'fsopts': "defaults,noatime" if not p['fsopts'] else p['fsopts']}
-
- if p['mountpoint'] == "/":
- for subvol in self.__instloop.subvolumes:
- if subvol['mountpoint'] == "/":
- continue
- s += "%(device)s %(mountpoint)s %(fstype)s %(fsopts)s 0 0\n" % {
- 'device': "/dev/%s%-d" % (p['disk_name'], p['num']),
- 'mountpoint': subvol['mountpoint'],
- 'fstype': p['fstype'],
- 'fsopts': "defaults,noatime" if not subvol['fsopts'] else subvol['fsopts']}
-
- s += "devpts /dev/pts devpts gid=5,mode=620 0 0\n"
- s += "tmpfs /dev/shm tmpfs defaults 0 0\n"
- s += "proc /proc proc defaults 0 0\n"
- s += "sysfs /sys sysfs defaults 0 0\n"
- return s
-
- def _create_mkinitrd_config(self):
- """write to tell which modules to be included in initrd"""
-
- mkinitrd = ""
- mkinitrd += "PROBE=\"no\"\n"
- mkinitrd += "MODULES+=\"ext3 ata_piix sd_mod libata scsi_mod\"\n"
- mkinitrd += "rootfs=\"ext3\"\n"
- mkinitrd += "rootopts=\"defaults\"\n"
-
- msger.debug("Writing mkinitrd config %s/etc/sysconfig/mkinitrd" \
- % self._instroot)
- os.makedirs(self._instroot + "/etc/sysconfig/",mode=644)
- cfg = open(self._instroot + "/etc/sysconfig/mkinitrd", "w")
- cfg.write(mkinitrd)
- cfg.close()
-
- def _get_parts(self):
- if not self.ks:
- raise CreatorError("Failed to get partition info, "
- "please check your kickstart setting.")
-
- # Set a default partition if no partition is given out
- if not self.ks.handler.partition.partitions:
- partstr = "part / --size 1900 --ondisk sda --fstype=ext3"
- args = partstr.split()
- pd = self.ks.handler.partition.parse(args[1:])
- if pd not in self.ks.handler.partition.partitions:
- self.ks.handler.partition.partitions.append(pd)
-
- # partitions list from kickstart file
- return kickstart.get_partitions(self.ks)
-
- def get_disk_names(self):
- """ Returns a list of physical target disk names (e.g., 'sdb') which
- will be created. """
-
- if self._disk_names:
- return self._disk_names
-
- #get partition info from ks handler
- parts = self._get_parts()
-
- for i in range(len(parts)):
- if parts[i].disk:
- disk_name = parts[i].disk
- else:
- raise CreatorError("Failed to create disks, no --ondisk "
- "specified in partition line of ks file")
-
- if parts[i].mountpoint and not parts[i].fstype:
- raise CreatorError("Failed to create disks, no --fstype "
- "specified for partition with mountpoint "
- "'%s' in the ks file")
-
- self._disk_names.append(disk_name)
-
- return self._disk_names
-
- def _full_name(self, name, extention):
- """ Construct full file name for a file we generate. """
- return "%s-%s.%s" % (self.name, name, extention)
-
- def _full_path(self, path, name, extention):
- """ Construct full file path to a file we generate. """
- return os.path.join(path, self._full_name(name, extention))
-
- #
- # Actual implemention
- #
- def _mount_instroot(self, base_on = None):
- parts = self._get_parts()
- self.__instloop = PartitionedMount(self._instroot)
-
- for p in parts:
- self.__instloop.add_partition(int(p.size),
- p.disk,
- p.mountpoint,
- p.fstype,
- p.label,
- fsopts = p.fsopts,
- boot = p.active,
- align = p.align,
- part_type = p.part_type)
-
- self.__instloop.layout_partitions(self._ptable_format)
-
- # Create the disks
- self.__imgdir = self._mkdtemp()
- for disk_name, disk in self.__instloop.disks.items():
- full_path = self._full_path(self.__imgdir, disk_name, "raw")
- msger.debug("Adding disk %s as %s with size %s bytes" \
- % (disk_name, full_path, disk['min_size']))
-
- disk_obj = fs_related.SparseLoopbackDisk(full_path,
- disk['min_size'])
- self.__disks[disk_name] = disk_obj
- self.__instloop.add_disk(disk_name, disk_obj)
-
- self.__instloop.mount()
- self._create_mkinitrd_config()
-
- def mount(self, base_on = None, cachedir = None):
- """
- This method calls the base class' 'mount()' method and then creates
- block device nodes corresponding to the image's partitions in the image
- itself. Namely, the image has /dev/loopX device corresponding to the
- entire image, and per-partition /dev/mapper/* devices.
-
- We copy these files to image's "/dev" directory in order to enable
- scripts which run in the image chroot environment to access own raw
- partitions. For example, this can be used to install the bootloader to
- the MBR (say, from an installer framework plugin).
- """
-
- def copy_devnode(src, dest):
- """A helper function for copying device nodes."""
-
- if not src:
- return
-
- stat_obj = os.stat(src)
- assert stat.S_ISBLK(stat_obj.st_mode)
-
- os.mknod(dest, stat_obj.st_mode,
- os.makedev(os.major(stat_obj.st_rdev),
- os.minor(stat_obj.st_rdev)))
- # os.mknod uses process umask may create a nod with different
- # permissions, so we use os.chmod to make sure permissions are
- # correct.
- os.chmod(dest, stat_obj.st_mode)
-
- BaseImageCreator.mount(self, base_on, cachedir)
-
- # Copy the disk loop devices
- for name in self.__disks.keys():
- loopdev = self.__disks[name].device
- copy_devnode(loopdev, self._instroot + loopdev)
-
- # Copy per-partition dm nodes
- os.mkdir(self._instroot + "/dev/mapper", os.stat("/dev/mapper").st_mode)
- for p in self.__instloop.partitions:
- copy_devnode(p['mapper_device'],
- self._instroot + p['mapper_device'])
- copy_devnode(p['mpath_device'],
- self._instroot + p['mpath_device'])
-
- def unmount(self):
- """
- Remove loop/dm device nodes which we created in 'mount()' and call the
- base class' 'unmount()' method.
- """
-
- for p in self.__instloop.partitions:
- if p['mapper_device']:
- path = self._instroot + p['mapper_device']
- if os.path.exists(path):
- os.unlink(path)
- if p['mpath_device']:
- path = self._instroot + p['mpath_device']
- if os.path.exists(path):
- os.unlink(path)
-
- path = self._instroot + "/dev/mapper"
- if os.path.exists(path):
- shutil.rmtree(path, ignore_errors=True)
-
- for name in self.__disks.keys():
- if self.__disks[name].device:
- path = self._instroot + self.__disks[name].device
- if os.path.exists(path):
- os.unlink(path)
-
- BaseImageCreator.unmount(self)
-
- def _get_required_packages(self):
- required_packages = BaseImageCreator._get_required_packages(self)
- if self._need_extlinux:
- if not self.target_arch or not self.target_arch.startswith("arm"):
- required_packages += ["syslinux", "syslinux-extlinux"]
- return required_packages
-
- def _get_excluded_packages(self):
- return BaseImageCreator._get_excluded_packages(self)
-
- def _get_syslinux_boot_config(self):
- rootdev = None
- root_part_uuid = None
- for p in self.__instloop.partitions:
- if p['mountpoint'] == "/":
- rootdev = "/dev/%s%-d" % (p['disk_name'], p['num'])
- root_part_uuid = p['partuuid']
-
- return (rootdev, root_part_uuid)
-
- def _create_syslinux_config(self):
-
- splash = os.path.join(self._instroot, "boot/extlinux")
- if os.path.exists(splash):
- splashline = "menu background splash.jpg"
- else:
- splashline = ""
-
- (rootdev, root_part_uuid) = self._get_syslinux_boot_config()
- options = self.ks.handler.bootloader.appendLine
-
- #XXX don't hardcode default kernel - see livecd code
- syslinux_conf = ""
- syslinux_conf += "prompt 0\n"
- syslinux_conf += "timeout 1\n"
- syslinux_conf += "\n"
- syslinux_conf += "default vesamenu.c32\n"
- syslinux_conf += "menu autoboot Starting %s...\n" % self.distro_name
- syslinux_conf += "menu hidden\n"
- syslinux_conf += "\n"
- syslinux_conf += "%s\n" % splashline
- syslinux_conf += "menu title Welcome to %s!\n" % self.distro_name
- syslinux_conf += "menu color border 0 #ffffffff #00000000\n"
- syslinux_conf += "menu color sel 7 #ffffffff #ff000000\n"
- syslinux_conf += "menu color title 0 #ffffffff #00000000\n"
- syslinux_conf += "menu color tabmsg 0 #ffffffff #00000000\n"
- syslinux_conf += "menu color unsel 0 #ffffffff #00000000\n"
- syslinux_conf += "menu color hotsel 0 #ff000000 #ffffffff\n"
- syslinux_conf += "menu color hotkey 7 #ffffffff #ff000000\n"
- syslinux_conf += "menu color timeout_msg 0 #ffffffff #00000000\n"
- syslinux_conf += "menu color timeout 0 #ffffffff #00000000\n"
- syslinux_conf += "menu color cmdline 0 #ffffffff #00000000\n"
-
- versions = []
- kernels = self._get_kernel_versions()
- symkern = "%s/boot/vmlinuz" % self._instroot
-
- if os.path.lexists(symkern):
- v = os.path.realpath(symkern).replace('%s-' % symkern, "")
- syslinux_conf += "label %s\n" % self.distro_name.lower()
- syslinux_conf += "\tmenu label %s (%s)\n" % (self.distro_name, v)
- syslinux_conf += "\tlinux ../vmlinuz\n"
- if self._ptable_format == 'msdos':
- rootstr = rootdev
- else:
- if not root_part_uuid:
- raise MountError("Cannot find the root GPT partition UUID")
- rootstr = "PARTUUID=%s" % root_part_uuid
- syslinux_conf += "\tappend ro root=%s %s\n" % (rootstr, options)
- syslinux_conf += "\tmenu default\n"
- else:
- for kernel in kernels:
- for version in kernels[kernel]:
- versions.append(version)
-
- footlabel = 0
- for v in versions:
- syslinux_conf += "label %s%d\n" \
- % (self.distro_name.lower(), footlabel)
- syslinux_conf += "\tmenu label %s (%s)\n" % (self.distro_name, v)
- syslinux_conf += "\tlinux ../vmlinuz-%s\n" % v
- syslinux_conf += "\tappend ro root=%s %s\n" \
- % (rootdev, options)
- if footlabel == 0:
- syslinux_conf += "\tmenu default\n"
- footlabel += 1;
-
- msger.debug("Writing syslinux config %s/boot/extlinux/extlinux.conf" \
- % self._instroot)
- cfg = open(self._instroot + "/boot/extlinux/extlinux.conf", "w")
- cfg.write(syslinux_conf)
- cfg.close()
-
- def _install_syslinux(self):
- for name in self.__disks.keys():
- loopdev = self.__disks[name].device
-
- # Set MBR
- mbrfile = "%s/usr/share/syslinux/" % self._instroot
- if self._ptable_format == 'gpt':
- mbrfile += "gptmbr.bin"
- else:
- mbrfile += "mbr.bin"
-
- msger.debug("Installing syslinux bootloader '%s' to %s" % \
- (mbrfile, loopdev))
-
- rc = runner.show(['dd', 'if=%s' % mbrfile, 'of=' + loopdev])
- if rc != 0:
- raise MountError("Unable to set MBR to %s" % loopdev)
-
-
- # Ensure all data is flushed to disk before doing syslinux install
- runner.quiet('sync')
-
- fullpathsyslinux = fs_related.find_binary_path("extlinux")
- rc = runner.show([fullpathsyslinux,
- "-i",
- "%s/boot/extlinux" % self._instroot])
- if rc != 0:
- raise MountError("Unable to install syslinux bootloader to %s" \
- % loopdev)
-
- def _create_bootconfig(self):
- #If syslinux is available do the required configurations.
- if self._need_extlinux \
- and os.path.exists("%s/usr/share/syslinux/" % (self._instroot)) \
- and os.path.exists("%s/boot/extlinux/" % (self._instroot)):
- self._create_syslinux_config()
- self._install_syslinux()
-
- def _unmount_instroot(self):
- if not self.__instloop is None:
- try:
- self.__instloop.cleanup()
- except MountError, err:
- msger.warning("%s" % err)
-
- def _resparse(self, size = None):
- return self.__instloop.resparse(size)
-
- def _get_post_scripts_env(self, in_chroot):
- env = BaseImageCreator._get_post_scripts_env(self, in_chroot)
-
- # Export the file-system UUIDs and partition UUIDs (AKA PARTUUIDs)
- for p in self.__instloop.partitions:
- env.update(self._set_part_env(p['ks_pnum'], "UUID", p['uuid']))
- env.update(self._set_part_env(p['ks_pnum'], "PARTUUID", p['partuuid']))
- env.update(self._set_part_env(p['ks_pnum'], "DEVNODE_NOW",
- p['mapper_device']))
- env.update(self._set_part_env(p['ks_pnum'], "DISK_DEVNODE_NOW",
- self.__disks[p['disk_name']].device))
-
- return env
-
- def _stage_final_image(self):
- """Stage the final system image in _outdir.
- write meta data
- """
- self._resparse()
- self.image_files.update({'disks': self.__disks.keys()})
-
- if not (self.compress_image or self.pack_to):
- for imgfile in os.listdir(self.__imgdir):
- if imgfile.endswith('.raw'):
- for disk in self.__disks.keys():
- if imgfile.find(disk) != -1:
- self.image_files.setdefault(disk, {}).update(
- {'image': imgfile})
- self.image_files.setdefault('image_files',
- []).append(imgfile)
-
- if self.compress_image:
- for imgfile in os.listdir(self.__imgdir):
- if imgfile.endswith('.raw') or imgfile.endswith('bin'):
- imgpath = os.path.join(self.__imgdir, imgfile)
- msger.info("Compressing image %s" % imgfile)
- compressing(imgpath, self.compress_image)
- if imgfile.endswith('.raw') and not self.pack_to:
- for disk in self.__disks.keys():
- if imgfile.find(disk) != -1:
- imgname = '%s.%s' % (imgfile, self.compress_image)
- self.image_files.setdefault(disk, {}).update(
- {'image': imgname})
- self.image_files.setdefault('image_files',
- []).append(imgname)
-
- if self.pack_to:
- dst = os.path.join(self._outdir, self.pack_to)
- msger.info("Pack all raw images to %s" % dst)
- packing(dst, self.__imgdir)
- self.image_files.update({'image_files': self.pack_to})
- else:
- msger.debug("moving disks to stage location")
- for imgfile in os.listdir(self.__imgdir):
- src = os.path.join(self.__imgdir, imgfile)
- dst = os.path.join(self._outdir, imgfile)
- msger.debug("moving %s to %s" % (src,dst))
- shutil.move(src,dst)
-
- self._write_image_xml()
-
- def _write_image_xml(self):
- imgarch = "i686"
- if self.target_arch and self.target_arch.startswith("arm"):
- imgarch = "arm"
- xml = "<image>\n"
-
- name_attributes = ""
- if self.appliance_version:
- name_attributes += " version='%s'" % self.appliance_version
- if self.appliance_release:
- name_attributes += " release='%s'" % self.appliance_release
- xml += " <name%s>%s</name>\n" % (name_attributes, self.name)
- xml += " <domain>\n"
- # XXX don't hardcode - determine based on the kernel we installed for
- # grub baremetal vs xen
- xml += " <boot type='hvm'>\n"
- xml += " <guest>\n"
- xml += " <arch>%s</arch>\n" % imgarch
- xml += " </guest>\n"
- xml += " <os>\n"
- xml += " <loader dev='hd'/>\n"
- xml += " </os>\n"
-
- i = 0
- for name in self.__disks.keys():
- full_name = self._full_name(name, self.__disk_format)
- xml += " <drive disk='%s' target='hd%s'/>\n" \
- % (full_name, chr(ord('a') + i))
- i = i + 1
-
- xml += " </boot>\n"
- xml += " <devices>\n"
- xml += " <vcpu>%s</vcpu>\n" % self.vcpu
- xml += " <memory>%d</memory>\n" %(self.vmem * 1024)
- for network in self.ks.handler.network.network:
- xml += " <interface/>\n"
- xml += " <graphics/>\n"
- xml += " </devices>\n"
- xml += " </domain>\n"
- xml += " <storage>\n"
-
- if self.checksum is True:
- for name in self.__disks.keys():
- diskpath = self._full_path(self._outdir, name, \
- self.__disk_format)
- full_name = self._full_name(name, self.__disk_format)
-
- msger.debug("Generating disk signature for %s" % full_name)
-
- xml += " <disk file='%s' use='system' format='%s'>\n" \
- % (full_name, self.__disk_format)
-
- hashes = misc.calc_hashes(diskpath, ('sha1', 'sha256'))
-
- xml += " <checksum type='sha1'>%s</checksum>\n" \
- % hashes[0]
- xml += " <checksum type='sha256'>%s</checksum>\n" \
- % hashes[1]
- xml += " </disk>\n"
- else:
- for name in self.__disks.keys():
- full_name = self._full_name(name, self.__disk_format)
- xml += " <disk file='%s' use='system' format='%s'/>\n" \
- % (full_name, self.__disk_format)
-
- xml += " </storage>\n"
- xml += "</image>\n"
-
- msger.debug("writing image XML to %s/%s.xml" %(self._outdir, self.name))
- cfg = open("%s/%s.xml" % (self._outdir, self.name), "w")
- cfg.write(xml)
- cfg.close()
-
- def generate_bmap(self):
- """ Generate block map file for the image. The idea is that while disk
- images we generate may be large (e.g., 4GiB), they may actually contain
- only little real data, e.g., 512MiB. This data are files, directories,
- file-system meta-data, partition table, etc. In other words, when
- flashing the image to the target device, you do not have to copy all the
- 4GiB of data, you can copy only 512MiB of it, which is 4 times faster.
-
- This function generates the block map file for an arbitrary image that
- mic has generated. The block map file is basically an XML file which
- contains a list of blocks which have to be copied to the target device.
- The other blocks are not used and there is no need to copy them. """
-
- if self.bmap_needed is None:
- return
-
- from mic.utils import BmapCreate
- msger.info("Generating the map file(s)")
-
- for name in self.__disks.keys():
- image = self._full_path(self.__imgdir, name, self.__disk_format)
- bmap_file = self._full_path(self._outdir, name, "bmap")
- self.image_files.setdefault(name, {}).update({'bmap': \
- os.path.basename(bmap_file)})
-
- msger.debug("Generating block map file '%s'" % bmap_file)
-
- try:
- creator = BmapCreate.BmapCreate(image, bmap_file)
- creator.generate()
- del creator
- except BmapCreate.Error as err:
- raise CreatorError("Failed to create bmap file: %s" % str(err))
-
- def create_manifest(self):
- if self.compress_image:
- self.image_files.update({'compress': self.compress_image})
- super(RawImageCreator, self).create_manifest()
+++ /dev/null
-# Copyright (c) 2012-2013 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License, version 2,
-# as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-
-"""
-This module implements the block map (bmap) creation functionality and provides
-the corresponding API in form of the 'BmapCreate' class.
-
-The idea is that while images files may generally be very large (e.g., 4GiB),
-they may nevertheless contain only little real data, e.g., 512MiB. This data
-are files, directories, file-system meta-data, partition table, etc. When
-copying the image to the target device, you do not have to copy all the 4GiB of
-data, you can copy only 512MiB of it, which is 4 times less, so copying should
-presumably be 4 times faster.
-
-The block map file is an XML file which contains a list of blocks which have to
-be copied to the target device. The other blocks are not used and there is no
-need to copy them. The XML file also contains some additional information like
-block size, image size, count of mapped blocks, etc. There are also many
-commentaries, so it is human-readable.
-
-The image has to be a sparse file. Generally, this means that when you generate
-this image file, you should start with a huge sparse file which contains a
-single hole spanning the entire file. Then you should partition it, write all
-the data (probably by means of loop-back mounting the image or parts of it),
-etc. The end result should be a sparse file where mapped areas represent useful
-parts of the image and holes represent useless parts of the image, which do not
-have to be copied when copying the image to the target device.
-
-This module uses the FIBMAP ioctl to detect holes.
-"""
-
-# Disable the following pylint recommendations:
-# * Too many instance attributes - R0902
-# * Too few public methods - R0903
-# pylint: disable=R0902,R0903
-
-import hashlib
-import logging
-from mic.utils.misc import human_size
-from mic.utils import Filemap
-
-# The bmap format version we generate.
-#
-# Changelog:
-# o 1.3 -> 2.0:
-# Support SHA256 and SHA512 checksums, in 1.3 only SHA1 was supported.
-# "BmapFileChecksum" is used instead of "BmapFileSHA1", and "chksum="
-# attribute is used instead "sha1=". Introduced "ChecksumType" tag. This is
-# an incompatible change.
-# Note, bmap format 1.4 is identical to 2.0. Version 1.4 was a mistake,
-# instead of incrementing the major version number, we incremented minor
-# version number. Unfortunately, the mistake slipped into bmap-tools version
-# 3.0, and was only fixed in bmap-tools v3.1.
-SUPPORTED_BMAP_VERSION = "2.0"
-
-_BMAP_START_TEMPLATE = \
-"""<?xml version="1.0" ?>
-<!-- This file contains the block map for an image file, which is basically
- a list of useful (mapped) block numbers in the image file. In other words,
- it lists only those blocks which contain data (boot sector, partition
- table, file-system metadata, files, directories, extents, etc). These
- blocks have to be copied to the target device. The other blocks do not
- contain any useful data and do not have to be copied to the target
- device.
-
- The block map an optimization which allows to copy or flash the image to
- the image quicker than copying of flashing the entire image. This is
- because with bmap less data is copied: <MappedBlocksCount> blocks instead
- of <BlocksCount> blocks.
-
- Besides the machine-readable data, this file contains useful commentaries
- which contain human-readable information like image size, percentage of
- mapped data, etc.
-
- The 'version' attribute is the block map file format version in the
- 'major.minor' format. The version major number is increased whenever an
- incompatible block map format change is made. The minor number changes
- in case of minor backward-compatible changes. -->
-
-<bmap version="%s">
- <!-- Image size in bytes: %s -->
- <ImageSize> %u </ImageSize>
-
- <!-- Size of a block in bytes -->
- <BlockSize> %u </BlockSize>
-
- <!-- Count of blocks in the image file -->
- <BlocksCount> %u </BlocksCount>
-
-"""
-
-class Error(Exception):
- """
- A class for exceptions generated by this module. We currently support only
- one type of exceptions, and we basically throw human-readable problem
- description in case of errors.
- """
- pass
-
-class BmapCreate(object):
- """
- This class implements the bmap creation functionality. To generate a bmap
- for an image (which is supposedly a sparse file), you should first create
- an instance of 'BmapCreate' and provide:
-
- * full path or a file-like object of the image to create bmap for
- * full path or a file object to use for writing the results to
-
- Then you should invoke the 'generate()' method of this class. It will use
- the FIEMAP ioctl to generate the bmap.
- """
-
- def __init__(self, image, bmap, chksum_type="sha256", log=None):
- """
- Initialize a class instance:
- * image - full path or a file-like object of the image to create bmap
- for
- * bmap - full path or a file object to use for writing the resulting
- bmap to
- * chksum - type of the check sum to use in the bmap file (all checksum
- types which python's "hashlib" module supports are allowed).
- * log - the logger object to use for printing messages.
- """
-
- self._log = log
- if self._log is None:
- self._log = logging.getLogger(__name__)
-
- self.image_size = None
- self.image_size_human = None
- self.block_size = None
- self.blocks_cnt = None
- self.mapped_cnt = None
- self.mapped_size = None
- self.mapped_size_human = None
- self.mapped_percent = None
-
- self._mapped_count_pos1 = None
- self._mapped_count_pos2 = None
- self._chksum_pos = None
-
- self._f_image_needs_close = False
- self._f_bmap_needs_close = False
-
- self._cs_type = chksum_type.lower()
- try:
- self._cs_len = len(hashlib.new(self._cs_type).hexdigest())
- except ValueError as err:
- raise Error("cannot initialize hash function \"%s\": %s" %
- (self._cs_type, err))
-
- if hasattr(image, "read"):
- self._f_image = image
- self._image_path = image.name
- else:
- self._image_path = image
- self._open_image_file()
-
- if hasattr(bmap, "read"):
- self._f_bmap = bmap
- self._bmap_path = bmap.name
- else:
- self._bmap_path = bmap
- self._open_bmap_file()
-
- try:
- self.filemap = Filemap.filemap(self._f_image, self._log)
- except (Filemap.Error, Filemap.ErrorNotSupp) as err:
- raise Error("cannot generate bmap: %s" % err)
-
- self.image_size = self.filemap.image_size
- self.image_size_human = human_size(self.image_size)
- if self.image_size == 0:
- raise Error("cannot generate bmap for zero-sized image file '%s'"
- % self._image_path)
-
- self.block_size = self.filemap.block_size
- self.blocks_cnt = self.filemap.blocks_cnt
-
- def __del__(self):
- """The class destructor which closes the opened files."""
- if self._f_image_needs_close:
- self._f_image.close()
- if self._f_bmap_needs_close:
- self._f_bmap.close()
-
- def _open_image_file(self):
- """Open the image file."""
- try:
- self._f_image = open(self._image_path, 'rb')
- except IOError as err:
- raise Error("cannot open image file '%s': %s"
- % (self._image_path, err))
-
- self._f_image_needs_close = True
-
- def _open_bmap_file(self):
- """Open the bmap file."""
- try:
- self._f_bmap = open(self._bmap_path, 'w+')
- except IOError as err:
- raise Error("cannot open bmap file '%s': %s"
- % (self._bmap_path, err))
-
- self._f_bmap_needs_close = True
-
- def _bmap_file_start(self):
- """
- A helper function which generates the starting contents of the block
- map file: the header comment, image size, block size, etc.
- """
-
- # We do not know the amount of mapped blocks at the moment, so just put
- # whitespaces instead of real numbers. Assume the longest possible
- # numbers.
-
- xml = _BMAP_START_TEMPLATE \
- % (SUPPORTED_BMAP_VERSION, self.image_size_human,
- self.image_size, self.block_size, self.blocks_cnt)
- xml += " <!-- Count of mapped blocks: "
-
- self._f_bmap.write(xml)
- self._mapped_count_pos1 = self._f_bmap.tell()
-
- xml = "%s or %s -->\n" % (' ' * len(self.image_size_human),
- ' ' * len("100.0%"))
- xml += " <MappedBlocksCount> "
-
- self._f_bmap.write(xml)
- self._mapped_count_pos2 = self._f_bmap.tell()
-
- xml = "%s </MappedBlocksCount>\n\n" % (' ' * len(str(self.blocks_cnt)))
-
- # pylint: disable=C0301
- xml += " <!-- Type of checksum used in this file -->\n"
- xml += " <ChecksumType> %s </ChecksumType>\n\n" % self._cs_type
-
- xml += " <!-- The checksum of this bmap file. When it is calculated, the value of\n"
- xml += " the checksum has be zero (all ASCII \"0\" symbols). -->\n"
- xml += " <BmapFileChecksum> "
-
- self._f_bmap.write(xml)
- self._chksum_pos = self._f_bmap.tell()
-
- xml = "0" * self._cs_len + " </BmapFileChecksum>\n\n"
- xml += " <!-- The block map which consists of elements which may either be a\n"
- xml += " range of blocks or a single block. The 'chksum' attribute\n"
- xml += " (if present) is the checksum of this blocks range. -->\n"
- xml += " <BlockMap>\n"
- # pylint: enable=C0301
-
- self._f_bmap.write(xml)
-
- def _bmap_file_end(self):
- """
- A helper function which generates the final parts of the block map
- file: the ending tags and the information about the amount of mapped
- blocks.
- """
-
- xml = " </BlockMap>\n"
- xml += "</bmap>\n"
-
- self._f_bmap.write(xml)
-
- self._f_bmap.seek(self._mapped_count_pos1)
- self._f_bmap.write("%s or %.1f%%"
- % (self.mapped_size_human, self.mapped_percent))
-
- self._f_bmap.seek(self._mapped_count_pos2)
- self._f_bmap.write("%u" % self.mapped_cnt)
-
- self._f_bmap.seek(0)
- hash_obj = hashlib.new(self._cs_type)
- hash_obj.update(self._f_bmap.read())
- chksum = hash_obj.hexdigest()
- self._f_bmap.seek(self._chksum_pos)
- self._f_bmap.write("%s" % chksum)
-
- def _calculate_chksum(self, first, last):
- """
- A helper function which calculates checksum for the range of blocks of
- the image file: from block 'first' to block 'last'.
- """
-
- start = first * self.block_size
- end = (last + 1) * self.block_size
-
- self._f_image.seek(start)
- hash_obj = hashlib.new(self._cs_type)
-
- chunk_size = 1024*1024
- to_read = end - start
- read = 0
-
- while read < to_read:
- if read + chunk_size > to_read:
- chunk_size = to_read - read
- chunk = self._f_image.read(chunk_size)
- hash_obj.update(chunk)
- read += chunk_size
-
- return hash_obj.hexdigest()
-
- def generate(self, include_checksums=True):
- """
- Generate bmap for the image file. If 'include_checksums' is 'True',
- also generate checksums for block ranges.
- """
-
- # Save image file position in order to restore it at the end
- image_pos = self._f_image.tell()
-
- self._bmap_file_start()
-
- # Generate the block map and write it to the XML block map
- # file as we go.
- self.mapped_cnt = 0
- for first, last in self.filemap.get_mapped_ranges(0, self.blocks_cnt):
- self.mapped_cnt += last - first + 1
- if include_checksums:
- chksum = self._calculate_chksum(first, last)
- chksum = " chksum=\"%s\"" % chksum
- else:
- chksum = ""
-
- if first != last:
- self._f_bmap.write(" <Range%s> %s-%s </Range>\n"
- % (chksum, first, last))
- else:
- self._f_bmap.write(" <Range%s> %s </Range>\n"
- % (chksum, first))
-
- self.mapped_size = self.mapped_cnt * self.block_size
- self.mapped_size_human = human_size(self.mapped_size)
- self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt
-
- self._bmap_file_end()
-
- try:
- self._f_bmap.flush()
- except IOError as err:
- raise Error("cannot flush the bmap file '%s': %s"
- % (self._bmap_path, err))
-
- self._f_image.seek(image_pos)
+++ /dev/null
-# Copyright (c) 2012 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License, version 2,
-# as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-
-"""
-This module implements python implements a way to get file block. Two methods
-are supported - the FIEMAP ioctl and the 'SEEK_HOLE / SEEK_DATA' features of
-the file seek syscall. The former is implemented by the 'FilemapFiemap' class,
-the latter is implemented by the 'FilemapSeek' class. Both classes provide the
-same API. The 'filemap' function automatically selects which class can be used
-and returns an instance of the class.
-"""
-
-# Disable the following pylint recommendations:
-# * Too many instance attributes (R0902)
-# pylint: disable=R0902
-
-import os
-import struct
-import array
-import fcntl
-import tempfile
-import logging
-from mic.utils.misc import get_block_size
-
-
-class ErrorNotSupp(Exception):
- """
- An exception of this type is raised when the 'FIEMAP' or 'SEEK_HOLE' feature
- is not supported either by the kernel or the file-system.
- """
- pass
-
-class Error(Exception):
- """A class for all the other exceptions raised by this module."""
- pass
-
-
-class _FilemapBase(object):
- """
- This is a base class for a couple of other classes in this module. This
- class simply performs the common parts of the initialization process: opens
- the image file, gets its size, etc. The 'log' parameter is the logger object
- to use for printing messages.
- """
-
- def __init__(self, image, log=None):
- """
- Initialize a class instance. The 'image' argument is full path to the
- file or file object to operate on.
- """
-
- self._log = log
- if self._log is None:
- self._log = logging.getLogger(__name__)
-
- self._f_image_needs_close = False
-
- if hasattr(image, "fileno"):
- self._f_image = image
- self._image_path = image.name
- else:
- self._image_path = image
- self._open_image_file()
-
- try:
- self.image_size = os.fstat(self._f_image.fileno()).st_size
- except IOError as err:
- raise Error("cannot get information about file '%s': %s"
- % (self._f_image.name, err))
-
- try:
- self.block_size = get_block_size(self._f_image)
- except IOError as err:
- raise Error("cannot get block size for '%s': %s"
- % (self._image_path, err))
-
- self.blocks_cnt = self.image_size + self.block_size - 1
- self.blocks_cnt /= self.block_size
-
- try:
- self._f_image.flush()
- except IOError as err:
- raise Error("cannot flush image file '%s': %s"
- % (self._image_path, err))
-
- try:
- os.fsync(self._f_image.fileno()),
- except OSError as err:
- raise Error("cannot synchronize image file '%s': %s "
- % (self._image_path, err.strerror))
-
- self._log.debug("opened image \"%s\"" % self._image_path)
- self._log.debug("block size %d, blocks count %d, image size %d"
- % (self.block_size, self.blocks_cnt, self.image_size))
-
- def __del__(self):
- """The class destructor which just closes the image file."""
- if self._f_image_needs_close:
- self._f_image.close()
-
- def _open_image_file(self):
- """Open the image file."""
- try:
- self._f_image = open(self._image_path, 'rb')
- except IOError as err:
- raise Error("cannot open image file '%s': %s"
- % (self._image_path, err))
-
- self._f_image_needs_close = True
-
- def block_is_mapped(self, block): # pylint: disable=W0613,R0201
- """
- This method has has to be implemented by child classes. It returns
- 'True' if block number 'block' of the image file is mapped and 'False'
- otherwise.
- """
-
- raise Error("the method is not implemented")
-
- def block_is_unmapped(self, block): # pylint: disable=W0613,R0201
- """
- This method has has to be implemented by child classes. It returns
- 'True' if block number 'block' of the image file is not mapped (hole)
- and 'False' otherwise.
- """
-
- raise Error("the method is not implemented")
-
- def get_mapped_ranges(self, start, count): # pylint: disable=W0613,R0201
- """
- This method has has to be implemented by child classes. This is a
- generator which yields ranges of mapped blocks in the file. The ranges
- are tuples of 2 elements: [first, last], where 'first' is the first
- mapped block and 'last' is the last mapped block.
-
- The ranges are yielded for the area of the file of size 'count' blocks,
- starting from block 'start'.
- """
-
- raise Error("the method is not implemented")
-
- def get_unmapped_ranges(self, start, count): # pylint: disable=W0613,R0201
- """
- This method has has to be implemented by child classes. Just like
- 'get_mapped_ranges()', but yields unmapped block ranges instead
- (holes).
- """
-
- raise Error("the method is not implemented")
-
-
-# The 'SEEK_HOLE' and 'SEEK_DATA' options of the file seek system call
-_SEEK_DATA = 3
-_SEEK_HOLE = 4
-
-def _lseek(file_obj, offset, whence):
- """This is a helper function which invokes 'os.lseek' for file object
- 'file_obj' and with specified 'offset' and 'whence'. The 'whence'
- argument is supposed to be either '_SEEK_DATA' or '_SEEK_HOLE'. When
- there is no more data or hole starting from 'offset', this function
- returns '-1'. Otherwise the data or hole position is returned."""
-
- try:
- return os.lseek(file_obj.fileno(), offset, whence)
- except OSError as err:
- # The 'lseek' system call returns the ENXIO if there is no data or
- # hole starting from the specified offset.
- if err.errno == os.errno.ENXIO:
- return -1
- elif err.errno == os.errno.EINVAL:
- raise ErrorNotSupp("the kernel or file-system does not support "
- "\"SEEK_HOLE\" and \"SEEK_DATA\"")
- else:
- raise
-
-class FilemapSeek(_FilemapBase):
- """
- This class uses the 'SEEK_HOLE' and 'SEEK_DATA' to find file block mapping.
- Unfortunately, the current implementation requires the caller to have write
- access to the image file.
- """
-
- def __init__(self, image, log=None):
- """Refer the '_FilemapBase' class for the documentation."""
-
- # Call the base class constructor first
- _FilemapBase.__init__(self, image, log)
- self._log.debug("FilemapSeek: initializing")
-
- self._probe_seek_hole()
-
- def _probe_seek_hole(self):
- """
- Check whether the system implements 'SEEK_HOLE' and 'SEEK_DATA'.
- Unfortunately, there seems to be no clean way for detecting this,
- because often the system just fakes them by just assuming that all
- files are fully mapped, so 'SEEK_HOLE' always returns EOF and
- 'SEEK_DATA' always returns the requested offset.
-
- I could not invent a better way of detecting the fake 'SEEK_HOLE'
- implementation than just to create a temporary file in the same
- directory where the image file resides. It would be nice to change this
- to something better.
- """
-
- directory = os.path.dirname(self._image_path)
-
- try:
- tmp_obj = tempfile.TemporaryFile("w+", dir=directory)
- except IOError as err:
- raise ErrorNotSupp("cannot create a temporary in \"%s\": %s"
- % (directory, err))
-
- try:
- os.ftruncate(tmp_obj.fileno(), self.block_size)
- except OSError as err:
- raise ErrorNotSupp("cannot truncate temporary file in \"%s\": %s"
- % (directory, err))
-
- offs = _lseek(tmp_obj, 0, _SEEK_HOLE)
- if offs != 0:
- # We are dealing with the stub 'SEEK_HOLE' implementation which
- # always returns EOF.
- self._log.debug("lseek(0, SEEK_HOLE) returned %d" % offs)
- raise ErrorNotSupp("the file-system does not support "
- "\"SEEK_HOLE\" and \"SEEK_DATA\" but only "
- "provides a stub implementation")
-
- tmp_obj.close()
-
- def block_is_mapped(self, block):
- """Refer the '_FilemapBase' class for the documentation."""
- offs = _lseek(self._f_image, block * self.block_size, _SEEK_DATA)
- if offs == -1:
- result = False
- else:
- result = (offs / self.block_size == block)
-
- self._log.debug("FilemapSeek: block_is_mapped(%d) returns %s"
- % (block, result))
- return result
-
- def block_is_unmapped(self, block):
- """Refer the '_FilemapBase' class for the documentation."""
- return not self.block_is_mapped(block)
-
- def _get_ranges(self, start, count, whence1, whence2):
- """
- This function implements 'get_mapped_ranges()' and
- 'get_unmapped_ranges()' depending on what is passed in the 'whence1'
- and 'whence2' arguments.
- """
-
- assert whence1 != whence2
- end = start * self.block_size
- limit = end + count * self.block_size
-
- while True:
- start = _lseek(self._f_image, end, whence1)
- if start == -1 or start >= limit or start == self.image_size:
- break
-
- end = _lseek(self._f_image, start, whence2)
- if end == -1 or end == self.image_size:
- end = self.blocks_cnt * self.block_size
- if end > limit:
- end = limit
-
- start_blk = start / self.block_size
- end_blk = end / self.block_size - 1
- self._log.debug("FilemapSeek: yielding range (%d, %d)"
- % (start_blk, end_blk))
- yield (start_blk, end_blk)
-
- def get_mapped_ranges(self, start, count):
- """Refer the '_FilemapBase' class for the documentation."""
- self._log.debug("FilemapSeek: get_mapped_ranges(%d, %d(%d))"
- % (start, count, start + count - 1))
- return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE)
-
- def get_unmapped_ranges(self, start, count):
- """Refer the '_FilemapBase' class for the documentation."""
- self._log.debug("FilemapSeek: get_unmapped_ranges(%d, %d(%d))"
- % (start, count, start + count - 1))
- return self._get_ranges(start, count, _SEEK_HOLE, _SEEK_DATA)
-
-
-# Below goes the FIEMAP ioctl implementation, which is not very readable
-# because it deals with the rather complex FIEMAP ioctl. To understand the
-# code, you need to know the FIEMAP interface, which is documented in the
-# "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources.
-
-# Format string for 'struct fiemap'
-_FIEMAP_FORMAT = "=QQLLLL"
-# sizeof(struct fiemap)
-_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT)
-# Format string for 'struct fiemap_extent'
-_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL"
-# sizeof(struct fiemap_extent)
-_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT)
-# The FIEMAP ioctl number
-_FIEMAP_IOCTL = 0xC020660B
-# This FIEMAP ioctl flag which instructs the kernel to sync the file before
-# reading the block map
-_FIEMAP_FLAG_SYNC = 0x00000001
-# Size of the buffer for 'struct fiemap_extent' elements which will be used
-# when invoking the FIEMAP ioctl. The larger is the buffer, the less times the
-# FIEMAP ioctl will be invoked.
-_FIEMAP_BUFFER_SIZE = 256 * 1024
-
-class FilemapFiemap(_FilemapBase):
- """
- This class provides API to the FIEMAP ioctl. Namely, it allows to iterate
- over all mapped blocks and over all holes.
-
- This class synchronizes the image file every time it invokes the FIEMAP
- ioctl in order to work-around early FIEMAP implementation kernel bugs.
- """
-
- def __init__(self, image, log=None):
- """
- Initialize a class instance. The 'image' argument is full the file
- object to operate on.
- """
-
- # Call the base class constructor first
- _FilemapBase.__init__(self, image, log)
- self._log.debug("FilemapFiemap: initializing")
-
- self._buf_size = _FIEMAP_BUFFER_SIZE
-
- # Calculate how many 'struct fiemap_extent' elements fit the buffer
- self._buf_size -= _FIEMAP_SIZE
- self._fiemap_extent_cnt = self._buf_size / _FIEMAP_EXTENT_SIZE
- assert self._fiemap_extent_cnt > 0
- self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE
- self._buf_size += _FIEMAP_SIZE
-
- # Allocate a mutable buffer for the FIEMAP ioctl
- self._buf = array.array('B', [0] * self._buf_size)
-
- # Check if the FIEMAP ioctl is supported
- self.block_is_mapped(0)
-
- def _invoke_fiemap(self, block, count):
- """
- Invoke the FIEMAP ioctl for 'count' blocks of the file starting from
- block number 'block'.
-
- The full result of the operation is stored in 'self._buf' on exit.
- Returns the unpacked 'struct fiemap' data structure in form of a python
- list (just like 'struct.upack()').
- """
-
- if self.blocks_cnt != 0 and (block < 0 or block >= self.blocks_cnt):
- raise Error("bad block number %d, should be within [0, %d]"
- % (block, self.blocks_cnt))
-
- # Initialize the 'struct fiemap' part of the buffer. We use the
- # '_FIEMAP_FLAG_SYNC' flag in order to make sure the file is
- # synchronized. The reason for this is that early FIEMAP
- # implementations had many bugs related to cached dirty data, and
- # synchronizing the file is a necessary work-around.
- struct.pack_into(_FIEMAP_FORMAT, self._buf, 0, block * self.block_size,
- count * self.block_size, _FIEMAP_FLAG_SYNC, 0,
- self._fiemap_extent_cnt, 0)
-
- try:
- fcntl.ioctl(self._f_image, _FIEMAP_IOCTL, self._buf, 1)
- except IOError as err:
- # Note, the FIEMAP ioctl is supported by the Linux kernel starting
- # from version 2.6.28 (year 2008).
- if err.errno == os.errno.EOPNOTSUPP:
- errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \
- "by the file-system"
- self._log.debug(errstr)
- raise ErrorNotSupp(errstr)
- if err.errno == os.errno.ENOTTY:
- errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \
- "by the kernel"
- self._log.debug(errstr)
- raise ErrorNotSupp(errstr)
- raise Error("the FIEMAP ioctl failed for '%s': %s"
- % (self._image_path, err))
-
- return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE])
-
- def block_is_mapped(self, block):
- """Refer the '_FilemapBase' class for the documentation."""
- struct_fiemap = self._invoke_fiemap(block, 1)
-
- # The 3rd element of 'struct_fiemap' is the 'fm_mapped_extents' field.
- # If it contains zero, the block is not mapped, otherwise it is
- # mapped.
- result = bool(struct_fiemap[3])
- self._log.debug("FilemapFiemap: block_is_mapped(%d) returns %s"
- % (block, result))
- return result
-
- def block_is_unmapped(self, block):
- """Refer the '_FilemapBase' class for the documentation."""
- return not self.block_is_mapped(block)
-
- def _unpack_fiemap_extent(self, index):
- """
- Unpack a 'struct fiemap_extent' structure object number 'index' from
- the internal 'self._buf' buffer.
- """
-
- offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index
- return struct.unpack(_FIEMAP_EXTENT_FORMAT,
- self._buf[offset : offset + _FIEMAP_EXTENT_SIZE])
-
- def _do_get_mapped_ranges(self, start, count):
- """
- Implements most the functionality for the 'get_mapped_ranges()'
- generator: invokes the FIEMAP ioctl, walks through the mapped extents
- and yields mapped block ranges. However, the ranges may be consecutive
- (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()' simply merges
- them.
- """
-
- block = start
- while block < start + count:
- struct_fiemap = self._invoke_fiemap(block, count)
-
- mapped_extents = struct_fiemap[3]
- if mapped_extents == 0:
- # No more mapped blocks
- return
-
- extent = 0
- while extent < mapped_extents:
- fiemap_extent = self._unpack_fiemap_extent(extent)
-
- # Start of the extent
- extent_start = fiemap_extent[0]
- # Starting block number of the extent
- extent_block = extent_start / self.block_size
- # Length of the extent
- extent_len = fiemap_extent[2]
- # Count of blocks in the extent
- extent_count = extent_len / self.block_size
-
- # Extent length and offset have to be block-aligned
- assert extent_start % self.block_size == 0
- assert extent_len % self.block_size == 0
-
- if extent_block > start + count - 1:
- return
-
- first = max(extent_block, block)
- last = min(extent_block + extent_count, start + count) - 1
- yield (first, last)
-
- extent += 1
-
- block = extent_block + extent_count
-
- def get_mapped_ranges(self, start, count):
- """Refer the '_FilemapBase' class for the documentation."""
- self._log.debug("FilemapFiemap: get_mapped_ranges(%d, %d(%d))"
- % (start, count, start + count - 1))
- iterator = self._do_get_mapped_ranges(start, count)
- first_prev, last_prev = iterator.next()
-
- for first, last in iterator:
- if last_prev == first - 1:
- last_prev = last
- else:
- self._log.debug("FilemapFiemap: yielding range (%d, %d)"
- % (first_prev, last_prev))
- yield (first_prev, last_prev)
- first_prev, last_prev = first, last
-
- self._log.debug("FilemapFiemap: yielding range (%d, %d)"
- % (first_prev, last_prev))
- yield (first_prev, last_prev)
-
- def get_unmapped_ranges(self, start, count):
- """Refer the '_FilemapBase' class for the documentation."""
- self._log.debug("FilemapFiemap: get_unmapped_ranges(%d, %d(%d))"
- % (start, count, start + count - 1))
- hole_first = start
- for first, last in self._do_get_mapped_ranges(start, count):
- if first > hole_first:
- self._log.debug("FilemapFiemap: yielding range (%d, %d)"
- % (hole_first, first - 1))
- yield (hole_first, first - 1)
-
- hole_first = last + 1
-
- if hole_first < start + count:
- self._log.debug("FilemapFiemap: yielding range (%d, %d)"
- % (hole_first, start + count - 1))
- yield (hole_first, start + count - 1)
-
-
-def filemap(image, log=None):
- """
- Create and return an instance of a Filemap class - 'FilemapFiemap' or
- 'FilemapSeek', depending on what the system we run on supports. If the
- FIEMAP ioctl is supported, an instance of the 'FilemapFiemap' class is
- returned. Otherwise, if 'SEEK_HOLE' is supported an instance of the
- 'FilemapSeek' class is returned. If none of these are supported, the
- function generates an 'Error' type exception.
- """
-
- try:
- return FilemapFiemap(image, log)
- except ErrorNotSupp:
- return FilemapSeek(image, log)
+++ /dev/null
-#!/usr/bin/python -tt
-#
-# Copyright (c) 2011 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-import os
-import shutil
-import re
-import tempfile
-
-from mic import chroot, msger, rt_util
-from mic.utils import misc, fs_related, errors, runner, cmdln
-from mic.conf import configmgr
-from mic.plugin import pluginmgr
-from mic.utils.partitionedfs import PartitionedMount
-
-import mic.imager.raw as raw
-
-from mic.pluginbase import ImagerPlugin
-class RawPlugin(ImagerPlugin):
- name = 'raw'
-
- @classmethod
- @cmdln.option("--compress-disk-image", dest="compress_image", type='choice',
- choices=("gz", "bz2"), default=None,
- help="Same with --compress-image")
- @cmdln.option("--compress-image", dest="compress_image", type='choice',
- choices=("gz", "bz2"), default = None,
- help="Compress all raw images before package")
- @cmdln.option("--generate-bmap", action="store_true", default = None,
- help="also generate the block map file")
- @cmdln.option("--fstab-entry", dest="fstab_entry", type='choice',
- choices=("name", "uuid"), default="uuid",
- help="Set fstab entry, 'name' means using device names, "
- "'uuid' means using filesystem uuid")
- def do_create(self, subcmd, opts, *args):
- """${cmd_name}: create raw image
-
- Usage:
- ${name} ${cmd_name} <ksfile> [OPTS]
-
- ${cmd_option_list}
- """
-
- if len(args) != 1:
- raise errors.Usage("Extra arguments given")
-
- creatoropts = configmgr.create
- ksconf = args[0]
-
- if creatoropts['runtime'] == "bootstrap":
- configmgr._ksconf = ksconf
- rt_util.bootstrap_mic()
-
- recording_pkgs = []
- if len(creatoropts['record_pkgs']) > 0:
- recording_pkgs = creatoropts['record_pkgs']
-
- if creatoropts['release'] is not None:
- if 'name' not in recording_pkgs:
- recording_pkgs.append('name')
- if 'vcs' not in recording_pkgs:
- recording_pkgs.append('vcs')
-
- configmgr._ksconf = ksconf
-
- # try to find the pkgmgr
- pkgmgr = None
- backends = pluginmgr.get_plugins('backend')
- if 'auto' == creatoropts['pkgmgr']:
- for key in configmgr.prefer_backends:
- if key in backends:
- pkgmgr = backends[key]
- break
- else:
- for key in backends.keys():
- if key == creatoropts['pkgmgr']:
- pkgmgr = backends[key]
- break
-
- if not pkgmgr:
- raise errors.CreatorError("Can't find backend: %s, "
- "available choices: %s" %
- (creatoropts['pkgmgr'],
- ','.join(backends.keys())))
-
- creator = raw.RawImageCreator(creatoropts, pkgmgr, opts.compress_image,
- opts.generate_bmap, opts.fstab_entry)
-
- if len(recording_pkgs) > 0:
- creator._recording_pkgs = recording_pkgs
-
- images = ["%s-%s.raw" % (creator.name, disk_name)
- for disk_name in creator.get_disk_names()]
- self.check_image_exists(creator.destdir,
- creator.pack_to,
- images,
- creatoropts['release'])
-
- try:
- creator.check_depend_tools()
- creator.mount(None, creatoropts["cachedir"])
- creator.install()
- creator.configure(creatoropts["repomd"])
- creator.copy_kernel()
- creator.unmount()
- creator.generate_bmap()
- creator.package(creatoropts["destdir"])
- creator.create_manifest()
- if creatoropts['release'] is not None:
- creator.release_output(ksconf, creatoropts['destdir'], creatoropts['release'])
- creator.print_outimage_info()
-
- except errors.CreatorError:
- raise
- finally:
- creator.cleanup()
-
- msger.info("Finished.")
- return 0
-
- @classmethod
- def do_chroot(cls, target, cmd=[]):
- img = target
- imgsize = misc.get_file_size(img) * 1024L * 1024L
- partedcmd = fs_related.find_binary_path("parted")
- disk = fs_related.SparseLoopbackDisk(img, imgsize)
- imgmnt = misc.mkdtemp()
- imgloop = PartitionedMount(imgmnt, skipformat = True)
- imgloop.add_disk('/dev/sdb', disk)
- img_fstype = "ext3"
-
- msger.info("Partition Table:")
- partnum = []
- for line in runner.outs([partedcmd, "-s", img, "print"]).splitlines():
- # no use strip to keep line output here
- if "Number" in line:
- msger.raw(line)
- if line.strip() and line.strip()[0].isdigit():
- partnum.append(line.strip()[0])
- msger.raw(line)
-
- rootpart = None
- if len(partnum) > 1:
- rootpart = msger.choice("please choose root partition", partnum)
-
- # Check the partitions from raw disk.
- # if choose root part, the mark it as mounted
- if rootpart:
- root_mounted = True
- else:
- root_mounted = False
- partition_mounts = 0
- for line in runner.outs([ partedcmd, "-s", img, "unit", "B", "print" ]).splitlines():
- line = line.strip()
-
- # Lines that start with number are the partitions,
- # because parted can be translated we can't refer to any text lines.
- if not line or not line[0].isdigit():
- continue
-
- # Some vars have extra , as list seperator.
- line = line.replace(",","")
-
- # Example of parted output lines that are handled:
- # Number Start End Size Type File system Flags
- # 1 512B 3400000511B 3400000000B primary
- # 2 3400531968B 3656384511B 255852544B primary linux-swap(v1)
- # 3 3656384512B 3720347647B 63963136B primary fat16 boot, lba
-
- partition_info = re.split("\s+", line)
-
- size = partition_info[3].split("B")[0]
-
- if len(partition_info) < 6 or partition_info[5] in ["boot"]:
- # No filesystem can be found from partition line. Assuming
- # btrfs, because that is the only MeeGo fs that parted does
- # not recognize properly.
- # TODO: Can we make better assumption?
- fstype = "btrfs"
- elif partition_info[5] in [ "ext2", "ext3", "ext4", "btrfs" ]:
- fstype = partition_info[5]
- elif partition_info[5] in [ "fat16", "fat32" ]:
- fstype = "vfat"
- elif "swap" in partition_info[5]:
- fstype = "swap"
- else:
- raise errors.CreatorError("Could not recognize partition fs type '%s'." %
- partition_info[5])
-
- if rootpart and rootpart == line[0]:
- mountpoint = '/'
- elif not root_mounted and fstype in [ "ext2", "ext3", "ext4", "btrfs" ]:
- # TODO: Check that this is actually the valid root partition from /etc/fstab
- mountpoint = "/"
- root_mounted = True
- elif fstype == "swap":
- mountpoint = "swap"
- else:
- # TODO: Assing better mount points for the rest of the partitions.
- partition_mounts += 1
- mountpoint = "/media/partition_%d" % partition_mounts
-
- if "boot" in partition_info:
- boot = True
- else:
- boot = False
-
- msger.verbose("Size: %s Bytes, fstype: %s, mountpoint: %s, boot: %s" %
- (size, fstype, mountpoint, boot))
- # TODO: add_partition should take bytes as size parameter.
- imgloop.add_partition((int)(size)/1024/1024, "/dev/sdb", mountpoint,
- fstype = fstype, boot = boot)
-
- try:
- imgloop.mount()
-
- except errors.MountError:
- imgloop.cleanup()
- raise
-
- try:
- if len(cmd) != 0:
- cmdline = ' '.join(cmd)
- else:
- cmdline = "/bin/bash"
- envcmd = fs_related.find_binary_inchroot("env", imgmnt)
- if envcmd:
- cmdline = "%s HOME=/root %s" % (envcmd, cmdline)
- chroot.chroot(imgmnt, None, cmdline)
- except:
- raise errors.CreatorError("Failed to chroot to %s." %img)
- finally:
- chroot.cleanup_after_chroot("img", imgloop, None, imgmnt)
-
- @classmethod
- def do_unpack(cls, srcimg):
- srcimgsize = (misc.get_file_size(srcimg)) * 1024L * 1024L
- srcmnt = misc.mkdtemp("srcmnt")
- disk = fs_related.SparseLoopbackDisk(srcimg, srcimgsize)
- srcloop = PartitionedMount(srcmnt, skipformat = True)
-
- srcloop.add_disk('/dev/sdb', disk)
- srcloop.add_partition(srcimgsize/1024/1024, "/dev/sdb", "/", "ext3", boot=False)
- try:
- srcloop.mount()
-
- except errors.MountError:
- srcloop.cleanup()
- raise
-
- image = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), "target.img")
- args = ['dd', "if=%s" % srcloop.partitions[0]['device'], "of=%s" % image]
-
- msger.info("`dd` image ...")
- rc = runner.show(args)
- srcloop.cleanup()
- shutil.rmtree(os.path.dirname(srcmnt), ignore_errors = True)
-
- if rc != 0:
- raise errors.CreatorError("Failed to dd")
- else:
- return image
projects / tools / mic.git / commitdiff