4 # Copyright (c) 2007 Red Hat Inc.
5 # Copyright (c) 2009, 2010, 2011 Intel, Inc.
7 # This program is free software; you can redistribute it and/or modify it
8 # under the terms of the GNU General Public License as published by the Free
9 # Software Foundation; version 2 of the License
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 # You should have received a copy of the GNU General Public License along
17 # with this program; if not, write to the Free Software Foundation, Inc., 59
18 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
20 from __future__ import with_statement
30 from datetime import datetime
34 from mic import kickstart
35 from mic import msger, __version__ as VERSION
36 from mic.utils.errors import CreatorError, KsError, Abort
37 from mic.utils import misc, grabber, runner, fs_related as fs
38 from mic.chroot import kill_proc_inchroot
39 from mic.archive import get_archive_suffixes
40 from mic.conf import configmgr
41 #post script max run time
44 class BaseImageCreator(object):
45 """Installs a system to a chroot directory.
47 ImageCreator is the simplest creator class available; it will install and
48 configure a system image according to the supplied kickstart file.
52 import mic.imgcreate as imgcreate
53 ks = imgcreate.read_kickstart("foo.ks")
54 imgcreate.ImageCreator(ks, "foo").create()
63 def __init__(self, createopts = None, pkgmgr = None):
64 """Initialize an ImageCreator instance.
66 ks -- a pykickstart.KickstartParser instance; this instance will be
67 used to drive the install by e.g. providing the list of packages
68 to be installed, the system configuration and %post scripts
70 name -- a name for the image; used for e.g. image filenames or
77 self.__builddir = None
78 self.__bindmounts = []
82 self.tmpdir = "/var/tmp/mic"
83 self.cachedir = "/var/tmp/mic/cache"
84 self.workdir = "/var/tmp/mic/build"
86 self.installerfw_prefix = "INSTALLERFW_"
87 self.target_arch = "noarch"
88 self.strict_mode = False
89 self._local_pkgs_path = None
92 self.multiple_partitions = False
95 # If the kernel is save to the destdir when copy_kernel cmd is called.
96 self._need_copy_kernel = False
97 # setup tmpfs tmpdir when enabletmpfs is True
98 self.enabletmpfs = False
101 # Mapping table for variables that have different names.
102 optmap = {"pkgmgr" : "pkgmgr_name",
103 "arch" : "target_arch",
104 "local_pkgs_path" : "_local_pkgs_path",
105 "copy_kernel" : "_need_copy_kernel",
106 "strict_mode" : "strict_mode",
109 # update setting from createopts
110 for key in createopts.keys():
115 setattr(self, option, createopts[key])
117 self.destdir = os.path.abspath(os.path.expanduser(self.destdir))
120 (tar, ext) = os.path.splitext(self.pack_to)
121 if ext in (".gz", ".bz2", ".lzo", ".bz") and tar.endswith(".tar"):
123 if ext not in get_archive_suffixes():
124 self.pack_to += ".tar"
126 self._dep_checks = ["ls", "bash", "cp", "echo", "modprobe"]
128 # Output image file names
130 # Output info related with manifest
131 self.image_files = {}
132 # A flag to generate checksum
133 self._genchecksum = False
135 self._alt_initrd_name = None
137 self._recording_pkgs = []
139 # available size in root fs, init to 0
140 self._root_fs_avail = 0
142 # Name of the disk image file that is created.
143 self._img_name = None
145 self.image_format = None
147 # Save qemu emulator file names in order to clean up it finally
148 self.qemu_emulators = []
150 # No ks provided when called by convertor, so skip the dependency check
152 # If we have btrfs partition we need to check necessary tools
153 for part in self.ks.handler.partition.partitions:
154 if part.fstype and part.fstype == "btrfs":
155 self._dep_checks.append("mkfs.btrfs")
158 if part.fstype == "cpio":
161 raise KsError("The '--fstype' in ks file need to set 'cpio' when you want to generate image by cpio.")
162 if len(self.ks.handler.partition.partitions) > 1:
163 self.multiple_partitions = True
166 if self.target_arch.startswith("arm"):
167 for dep in self._dep_checks:
168 if dep == "extlinux":
169 self._dep_checks.remove(dep)
171 if not os.path.exists("/usr/bin/qemu-arm") or \
172 not misc.is_statically_linked("/usr/bin/qemu-arm"):
173 self._dep_checks.append("qemu-arm-static")
175 if os.path.exists("/proc/sys/vm/vdso_enabled"):
176 vdso_fh = open("/proc/sys/vm/vdso_enabled","r")
177 vdso_value = vdso_fh.read().strip()
179 if (int)(vdso_value) == 1:
180 msger.warning("vdso is enabled on your host, which might "
181 "cause problems with arm emulations.\n"
182 "\tYou can disable vdso with following command before "
183 "starting image build:\n"
184 "\techo 0 | sudo tee /proc/sys/vm/vdso_enabled")
185 elif self.target_arch == "mipsel":
186 for dep in self._dep_checks:
187 if dep == "extlinux":
188 self._dep_checks.remove(dep)
190 if not os.path.exists("/usr/bin/qemu-mipsel") or \
191 not misc.is_statically_linked("/usr/bin/qemu-mipsel"):
192 self._dep_checks.append("qemu-mipsel-static")
194 if os.path.exists("/proc/sys/vm/vdso_enabled"):
195 vdso_fh = open("/proc/sys/vm/vdso_enabled","r")
196 vdso_value = vdso_fh.read().strip()
198 if (int)(vdso_value) == 1:
199 msger.warning("vdso is enabled on your host, which might "
200 "cause problems with mipsel emulations.\n"
201 "\tYou can disable vdso with following command before "
202 "starting image build:\n"
203 "\techo 0 | sudo tee /proc/sys/vm/vdso_enabled")
205 # make sure the specified tmpdir and cachedir exist
206 if not os.path.exists(self.tmpdir):
207 os.makedirs(self.tmpdir)
208 if not os.path.exists(self.cachedir):
209 os.makedirs(self.cachedir)
215 def __get_instroot(self):
216 if self.__builddir is None:
217 raise CreatorError("_instroot is not valid before calling mount()")
218 return self.__builddir + "/install_root"
219 _instroot = property(__get_instroot)
220 """The location of the install root directory.
222 This is the directory into which the system is installed. Subclasses may
223 mount a filesystem image here or copy files to/from here.
225 Note, this directory does not exist before ImageCreator.mount() is called.
227 Note also, this is a read-only attribute.
231 def __get_outdir(self):
232 if self.__builddir is None:
233 raise CreatorError("_outdir is not valid before calling mount()")
234 return self.__builddir + "/out"
235 _outdir = property(__get_outdir)
236 """The staging location for the final image.
238 This is where subclasses should stage any files that are part of the final
239 image. ImageCreator.package() will copy any files found here into the
240 requested destination directory.
242 Note, this directory does not exist before ImageCreator.mount() is called.
244 Note also, this is a read-only attribute.
250 # Hooks for subclasses
252 def _mount_instroot(self, base_on = None):
253 """Mount or prepare the install root directory.
255 This is the hook where subclasses may prepare the install root by e.g.
256 mounting creating and loopback mounting a filesystem image to
259 There is no default implementation.
261 base_on -- this is the value passed to mount() and can be interpreted
262 as the subclass wishes; it might e.g. be the location of
263 a previously created ISO containing a system image.
268 def _unmount_instroot(self):
269 """Undo anything performed in _mount_instroot().
271 This is the hook where subclasses must undo anything which was done
272 in _mount_instroot(). For example, if a filesystem image was mounted
273 onto _instroot, it should be unmounted here.
275 There is no default implementation.
280 def _create_bootconfig(self):
281 """Configure the image so that it's bootable.
283 This is the hook where subclasses may prepare the image for booting by
284 e.g. creating an initramfs and bootloader configuration.
286 This hook is called while the install root is still mounted, after the
287 packages have been installed and the kickstart configuration has been
288 applied, but before the %post scripts have been executed.
290 There is no default implementation.
295 def _stage_final_image(self):
296 """Stage the final system image in _outdir.
298 This is the hook where subclasses should place the image in _outdir
299 so that package() can copy it to the requested destination directory.
301 By default, this moves the install root into _outdir.
304 shutil.move(self._instroot, self._outdir + "/" + self.name)
306 def get_installed_packages(self):
307 return self._pkgs_content.keys()
309 def _save_recording_pkgs(self, destdir):
310 """Save the list or content of installed packages to file.
312 pkgs = self._pkgs_content.keys()
313 pkgs.sort() # inplace op
315 if not os.path.exists(destdir):
319 if 'vcs' in self._recording_pkgs:
320 vcslst = ["%s %s" % (k, v) for (k, v) in self._pkgs_vcsinfo.items()]
321 content = '\n'.join(sorted(vcslst))
322 elif 'name' in self._recording_pkgs:
323 content = '\n'.join(pkgs)
325 namefile = os.path.join(destdir, self.name + '.packages')
326 f = open(namefile, "w")
329 self.outimage.append(namefile);
331 # if 'content', save more details
332 if 'content' in self._recording_pkgs:
333 contfile = os.path.join(destdir, self.name + '.files')
334 f = open(contfile, "w")
339 pkgcont = self._pkgs_content[pkg]
341 content += '\n '.join(pkgcont)
347 self.outimage.append(contfile)
349 if 'license' in self._recording_pkgs:
350 licensefile = os.path.join(destdir, self.name + '.license')
351 f = open(licensefile, "w")
353 f.write('Summary:\n')
354 for license in reversed(sorted(self._pkgs_license, key=\
355 lambda license: len(self._pkgs_license[license]))):
356 f.write(" - %s: %s\n" \
357 % (license, len(self._pkgs_license[license])))
359 f.write('\nDetails:\n')
360 for license in reversed(sorted(self._pkgs_license, key=\
361 lambda license: len(self._pkgs_license[license]))):
362 f.write(" - %s:\n" % (license))
363 for pkg in sorted(self._pkgs_license[license]):
364 f.write(" - %s\n" % (pkg))
368 self.outimage.append(licensefile)
370 def _get_required_packages(self):
371 """Return a list of required packages.
373 This is the hook where subclasses may specify a set of packages which
374 it requires to be installed.
376 This returns an empty list by default.
378 Note, subclasses should usually chain up to the base class
379 implementation of this hook.
384 def _get_excluded_packages(self):
385 """Return a list of excluded packages.
387 This is the hook where subclasses may specify a set of packages which
388 it requires _not_ to be installed.
390 This returns an empty list by default.
392 Note, subclasses should usually chain up to the base class
393 implementation of this hook.
398 def _get_local_packages(self):
399 """Return a list of rpm path to be local installed.
401 This is the hook where subclasses may specify a set of rpms which
402 it requires to be installed locally.
404 This returns an empty list by default.
406 Note, subclasses should usually chain up to the base class
407 implementation of this hook.
410 if self._local_pkgs_path:
411 if os.path.isdir(self._local_pkgs_path):
413 os.path.join(self._local_pkgs_path, '*.rpm'))
414 elif os.path.splitext(self._local_pkgs_path)[-1] == '.rpm':
415 return [self._local_pkgs_path]
419 def _get_fstab(self):
420 """Return the desired contents of /etc/fstab.
422 This is the hook where subclasses may specify the contents of
423 /etc/fstab by returning a string containing the desired contents.
425 A sensible default implementation is provided.
428 s = "/dev/root / %s %s 0 0\n" \
430 "defaults,noatime" if not self._fsopts else self._fsopts)
431 s += self._get_fstab_special()
434 def _get_fstab_special(self):
435 s = "devpts /dev/pts devpts gid=5,mode=620 0 0\n"
436 s += "tmpfs /dev/shm tmpfs defaults 0 0\n"
437 s += "proc /proc proc defaults 0 0\n"
438 s += "sysfs /sys sysfs defaults 0 0\n"
441 def _set_part_env(self, pnum, prop, value):
442 """ This is a helper function which generates an environment variable
443 for a property "prop" with value "value" of a partition number "pnum".
445 The naming convention is:
446 * Variables start with INSTALLERFW_PART
447 * Then goes the partition number, the order is the same as
448 specified in the KS file
449 * Then goes the property name
457 name = self.installerfw_prefix + ("PART%d_" % pnum) + prop
458 return { name : value }
460 def _get_post_scripts_env(self, in_chroot):
461 """Return an environment dict for %post scripts.
463 This is the hook where subclasses may specify some environment
464 variables for %post scripts by return a dict containing the desired
467 in_chroot -- whether this %post script is to be executed chroot()ed
474 for p in kickstart.get_partitions(self.ks):
475 env.update(self._set_part_env(pnum, "SIZE", p.size))
476 env.update(self._set_part_env(pnum, "MOUNTPOINT", p.mountpoint))
477 env.update(self._set_part_env(pnum, "FSTYPE", p.fstype))
478 env.update(self._set_part_env(pnum, "LABEL", p.label))
479 env.update(self._set_part_env(pnum, "FSOPTS", p.fsopts))
480 env.update(self._set_part_env(pnum, "BOOTFLAG", p.active))
481 env.update(self._set_part_env(pnum, "ALIGN", p.align))
482 env.update(self._set_part_env(pnum, "TYPE_ID", p.part_type))
483 env.update(self._set_part_env(pnum, "UUID", p.uuid))
484 env.update(self._set_part_env(pnum, "DEVNODE",
485 "/dev/%s%d" % (p.disk, pnum + 1)))
486 env.update(self._set_part_env(pnum, "DISK_DEVNODE",
491 env[self.installerfw_prefix + "PART_COUNT"] = str(pnum)
493 # Partition table format
494 ptable_format = self.ks.handler.bootloader.ptable
495 env[self.installerfw_prefix + "PTABLE_FORMAT"] = ptable_format
497 # The kerned boot parameters
498 kernel_opts = self.ks.handler.bootloader.appendLine
499 env[self.installerfw_prefix + "KERNEL_OPTS"] = kernel_opts
501 # Name of the image creation tool
502 env[self.installerfw_prefix + "INSTALLER_NAME"] = "mic"
504 # The real current location of the mounted file-systems
508 mount_prefix = self._instroot
509 env[self.installerfw_prefix + "MOUNT_PREFIX"] = mount_prefix
511 # These are historical variables which lack the common name prefix
513 env["INSTALL_ROOT"] = self._instroot
514 env["IMG_NAME"] = self._name
518 def __get_imgname(self):
520 _name = property(__get_imgname)
521 """The name of the image file.
525 def _get_kernel_versions(self):
526 """Return a dict detailing the available kernel types/versions.
528 This is the hook where subclasses may override what kernel types and
529 versions should be available for e.g. creating the booloader
532 A dict should be returned mapping the available kernel types to a list
533 of the available versions for those kernels.
535 The default implementation uses rpm to iterate over everything
536 providing 'kernel', finds /boot/vmlinuz-* and returns the version
537 obtained from the vmlinuz filename. (This can differ from the kernel
538 RPM's n-v-r in the case of e.g. xen)
541 def get_kernel_versions(instroot):
544 files = glob.glob(instroot + "/boot/vmlinuz-*")
546 version = os.path.basename(file)[8:]
549 versions.add(version)
550 ret["kernel"] = list(versions)
553 def get_version(header):
555 for f in header['filenames']:
556 if f.startswith('/boot/vmlinuz-'):
561 return get_kernel_versions(self._instroot)
563 ts = rpm.TransactionSet(self._instroot)
566 for header in ts.dbMatch('provides', 'kernel'):
567 version = get_version(header)
571 name = header['name']
573 ret[name] = [version]
574 elif not version in ret[name]:
575 ret[name].append(version)
581 # Helpers for subclasses
583 def _do_bindmounts(self):
584 """Mount various system directories onto _instroot.
586 This method is called by mount(), but may also be used by subclasses
587 in order to re-mount the bindmounts after modifying the underlying
591 for b in self.__bindmounts:
594 def _undo_bindmounts(self):
595 """Unmount the bind-mounted system directories from _instroot.
597 This method is usually only called by unmount(), but may also be used
598 by subclasses in order to gain access to the filesystem obscured by
599 the bindmounts - e.g. in order to create device nodes on the image
603 self.__bindmounts.reverse()
604 for b in self.__bindmounts:
608 """Chroot into the install root.
610 This method may be used by subclasses when executing programs inside
611 the install root e.g.
613 subprocess.call(["/bin/ls"], preexec_fn = self.chroot)
616 os.chroot(self._instroot)
619 def _mkdtemp(self, prefix = "tmp-"):
620 """Create a temporary directory.
622 This method may be used by subclasses to create a temporary directory
623 for use in building the final image - e.g. a subclass might create
624 a temporary directory in order to bundle a set of files into a package.
626 The subclass may delete this directory if it wishes, but it will be
627 automatically deleted by cleanup().
629 The absolute path to the temporary directory is returned.
631 Note, this method should only be called after mount() has been called.
633 prefix -- a prefix which should be used when creating the directory;
637 self.__ensure_builddir()
638 return tempfile.mkdtemp(dir = self.__builddir, prefix = prefix)
640 def _mkstemp(self, prefix = "tmp-"):
641 """Create a temporary file.
643 This method may be used by subclasses to create a temporary file
644 for use in building the final image - e.g. a subclass might need
645 a temporary location to unpack a compressed file.
647 The subclass may delete this file if it wishes, but it will be
648 automatically deleted by cleanup().
650 A tuple containing a file descriptor (returned from os.open() and the
651 absolute path to the temporary directory is returned.
653 Note, this method should only be called after mount() has been called.
655 prefix -- a prefix which should be used when creating the file;
659 self.__ensure_builddir()
660 return tempfile.mkstemp(dir = self.__builddir, prefix = prefix)
662 def _mktemp(self, prefix = "tmp-"):
663 """Create a temporary file.
665 This method simply calls _mkstemp() and closes the returned file
668 The absolute path to the temporary file is returned.
670 Note, this method should only be called after mount() has been called.
672 prefix -- a prefix which should be used when creating the file;
677 (f, path) = self._mkstemp(prefix)
683 # Actual implementation
685 def __ensure_builddir(self):
686 if not self.__builddir is None:
690 self.workdir = os.path.join(self.tmpdir, "build")
691 if not os.path.exists(self.workdir):
692 os.makedirs(self.workdir)
693 self.__builddir = tempfile.mkdtemp(dir = self.workdir,
694 prefix = "imgcreate-")
695 except OSError, (err, msg):
696 raise CreatorError("Failed create build directory in %s: %s" %
699 def get_cachedir(self, cachedir = None):
703 self.__ensure_builddir()
705 self.cachedir = cachedir
707 self.cachedir = self.__builddir + "/mic-cache"
708 fs.makedirs(self.cachedir)
711 def __sanity_check(self):
712 """Ensure that the config we've been given is same."""
713 if not (kickstart.get_packages(self.ks) or
714 kickstart.get_groups(self.ks)):
715 raise CreatorError("No packages or groups specified")
717 kickstart.convert_method_to_repo(self.ks)
719 if not kickstart.get_repos(self.ks):
720 raise CreatorError("No repositories specified")
722 def __write_fstab(self):
723 if kickstart.use_installerfw(self.ks, "fstab"):
724 # The fstab file will be generated by installer framework scripts
727 fstab_contents = self._get_fstab()
729 fstab = open(self._instroot + "/etc/fstab", "w")
730 fstab.write(fstab_contents)
733 def __create_minimal_dev(self):
734 """Create a minimal /dev so that we don't corrupt the host /dev"""
735 origumask = os.umask(0000)
736 devices = (('null', 1, 3, 0666),
737 ('urandom',1, 9, 0666),
738 ('random', 1, 8, 0666),
739 ('full', 1, 7, 0666),
740 ('ptmx', 5, 2, 0666),
742 ('zero', 1, 5, 0666))
744 links = (("/proc/self/fd", "/dev/fd"),
745 ("/proc/self/fd/0", "/dev/stdin"),
746 ("/proc/self/fd/1", "/dev/stdout"),
747 ("/proc/self/fd/2", "/dev/stderr"))
749 for (node, major, minor, perm) in devices:
750 if not os.path.exists(self._instroot + "/dev/" + node):
751 os.mknod(self._instroot + "/dev/" + node,
753 os.makedev(major,minor))
755 for (src, dest) in links:
756 if not os.path.exists(self._instroot + dest):
757 os.symlink(src, self._instroot + dest)
761 def __setup_tmpdir(self):
762 if not self.enabletmpfs:
765 runner.show('mount -t tmpfs -o size=4G tmpfs %s' % self.workdir)
767 def __clean_tmpdir(self):
768 if not self.enabletmpfs:
771 runner.show('umount -l %s' % self.workdir)
774 #Add tpk-install option
775 createopts = configmgr.create
776 if createopts['tpk_install']:
777 path = createopts['tpk_install']
778 file_list = os.listdir(path)
780 sub = os.path.splitext(f)[1]
782 raise CreatorError("Not all files in the path: "+path +" is tpk")
784 tpk_dir = "/usr/apps/.preload-tpk"
785 fs.makedirs(self._instroot + "/usr/apps")
786 fs.makedirs(self._instroot + tpk_dir)
788 shutil.copy(path+"/"+f,self._instroot + tpk_dir)
790 def mount(self, base_on = None, cachedir = None):
791 """Setup the target filesystem in preparation for an install.
793 This function sets up the filesystem which the ImageCreator will
794 install into and configure. The ImageCreator class merely creates an
795 install root directory, bind mounts some system directories (e.g. /dev)
796 and writes out /etc/fstab. Other subclasses may also e.g. create a
797 sparse file, format it and loopback mount it to the install root.
799 base_on -- a previous install on which to base this install; defaults
800 to None, causing a new image to be created
802 cachedir -- a directory in which to store the Yum cache; defaults to
803 None, causing a new cache to be created; by setting this
804 to another directory, the same cache can be reused across
808 self.__setup_tmpdir()
809 self.__ensure_builddir()
811 # prevent popup dialog in Ubuntu(s)
812 misc.hide_loopdev_presentation()
814 fs.makedirs(self._instroot)
815 fs.makedirs(self._outdir)
817 self._mount_instroot(base_on)
819 for d in ("/dev/pts",
826 fs.makedirs(self._instroot + d)
828 if self.target_arch and self.target_arch.startswith("arm") or \
829 self.target_arch == "aarch64":
830 self.qemu_emulators = misc.setup_qemu_emulator(self._instroot,
833 self.get_cachedir(cachedir)
835 # bind mount system directories into _instroot
836 for (f, dest) in [("/sys", None),
838 ("/proc/sys/fs/binfmt_misc", None),
840 self.__bindmounts.append(
842 f, self._instroot, dest))
844 self._do_bindmounts()
846 self.__create_minimal_dev()
848 if os.path.exists(self._instroot + "/etc/mtab"):
849 os.unlink(self._instroot + "/etc/mtab")
850 os.symlink("../proc/mounts", self._instroot + "/etc/mtab")
854 # get size of available space in 'instroot' fs
855 self._root_fs_avail = misc.get_filesystem_avail(self._instroot)
859 """Unmounts the target filesystem.
861 The ImageCreator class detaches the system from the install root, but
862 other subclasses may also detach the loopback mounted filesystem image
863 from the install root.
867 mtab = self._instroot + "/etc/mtab"
868 if not os.path.islink(mtab):
869 os.unlink(self._instroot + "/etc/mtab")
871 for qemu_emulator in self.qemu_emulators:
872 os.unlink(self._instroot + qemu_emulator)
876 self._undo_bindmounts()
878 """ Clean up yum garbage """
880 instroot_pdir = os.path.dirname(self._instroot + self._instroot)
881 if os.path.exists(instroot_pdir):
882 shutil.rmtree(instroot_pdir, ignore_errors = True)
883 yumlibdir = self._instroot + "/var/lib/yum"
884 if os.path.exists(yumlibdir):
885 shutil.rmtree(yumlibdir, ignore_errors = True)
889 self._unmount_instroot()
891 # reset settings of popup dialog in Ubuntu(s)
892 misc.unhide_loopdev_presentation()
896 """Unmounts the target filesystem and deletes temporary files.
898 This method calls unmount() and then deletes any temporary files and
899 directories that were created on the host system while building the
902 Note, make sure to call this method once finished with the creator
903 instance in order to ensure no stale files are left on the host e.g.:
905 creator = ImageCreator(ks, name)
912 if not self.__builddir:
915 kill_proc_inchroot(self._instroot)
919 shutil.rmtree(self.__builddir, ignore_errors = True)
920 self.__builddir = None
922 self.__clean_tmpdir()
924 def __is_excluded_pkg(self, pkg):
925 if pkg in self._excluded_pkgs:
926 self._excluded_pkgs.remove(pkg)
929 for xpkg in self._excluded_pkgs:
930 if xpkg.endswith('*'):
931 if pkg.startswith(xpkg[:-1]):
933 elif xpkg.startswith('*'):
934 if pkg.endswith(xpkg[1:]):
939 def __select_packages(self, pkg_manager):
941 for pkg in self._required_pkgs:
942 e = pkg_manager.selectPackage(pkg)
944 if kickstart.ignore_missing(self.ks):
945 skipped_pkgs.append(pkg)
946 elif self.__is_excluded_pkg(pkg):
947 skipped_pkgs.append(pkg)
949 raise CreatorError("Failed to find package '%s' : %s" %
952 for pkg in skipped_pkgs:
953 msger.warning("Skipping missing package '%s'" % (pkg,))
955 def __select_groups(self, pkg_manager):
957 for group in self._required_groups:
958 e = pkg_manager.selectGroup(group.name, group.include)
960 if kickstart.ignore_missing(self.ks):
961 skipped_groups.append(group)
963 raise CreatorError("Failed to find group '%s' : %s" %
966 for group in skipped_groups:
967 msger.warning("Skipping missing group '%s'" % (group.name,))
969 def __deselect_packages(self, pkg_manager):
970 for pkg in self._excluded_pkgs:
971 pkg_manager.deselectPackage(pkg)
973 """def __localinst_packages(self, pkg_manager):
974 for rpm_path in self._get_local_packages():
975 pkg_manager.installLocal(rpm_path)"""
977 def __preinstall_packages(self, pkg_manager):
981 self._preinstall_pkgs = kickstart.get_pre_packages(self.ks)
982 for pkg in self._preinstall_pkgs:
983 pkg_manager.preInstall(pkg)
985 def __check_packages(self, pkg_manager):
986 for pkg in self.check_pkgs:
987 pkg_manager.checkPackage(pkg)
989 def __attachment_packages(self, pkg_manager):
993 self._attachment = []
994 for item in kickstart.get_attachment(self.ks):
995 if item.startswith('/'):
996 fpaths = os.path.join(self._instroot, item.lstrip('/'))
997 for fpath in glob.glob(fpaths):
998 self._attachment.append(fpath)
1001 filelist = pkg_manager.getFilelist(item)
1003 # found rpm in rootfs
1004 for pfile in pkg_manager.getFilelist(item):
1005 fpath = os.path.join(self._instroot, pfile.lstrip('/'))
1006 self._attachment.append(fpath)
1009 # try to retrieve rpm file
1010 (url, proxies) = pkg_manager.package_url(item)
1012 msger.warning("Can't get url from repo for %s" % item)
1014 fpath = os.path.join(self.cachedir, os.path.basename(url))
1015 if not os.path.exists(fpath):
1018 fpath = grabber.myurlgrab(url.full, fpath, proxies, None)
1019 except CreatorError:
1022 tmpdir = self._mkdtemp()
1023 misc.extract_rpm(fpath, tmpdir)
1024 for (root, dirs, files) in os.walk(tmpdir):
1026 fpath = os.path.join(root, fname)
1027 self._attachment.append(fpath)
1029 def install(self, repo_urls=None):
1030 """Install packages into the install root.
1032 This function installs the packages listed in the supplied kickstart
1033 into the install root. By default, the packages are installed from the
1034 repository URLs specified in the kickstart.
1036 repo_urls -- a dict which maps a repository name to a repository;
1037 if supplied, this causes any repository URLs specified in
1038 the kickstart to be overridden.
1041 def checkScriptletError(dirname, suffix):
1042 if os.path.exists(dirname):
1043 list = os.listdir(dirname)
1045 filepath = os.path.join(dirname, line)
1046 if os.path.isfile(filepath) and 0 < line.find(suffix):
1053 def showErrorInfo(filepath):
1054 if os.path.isfile(filepath):
1055 for line in open(filepath):
1056 msger.info("The error install package info: %s" % line)
1058 msger.info("%s is not found." % filepath)
1060 def get_ssl_verify(ssl_verify=None):
1061 if ssl_verify is not None:
1062 return not ssl_verify.lower().strip() == 'no'
1064 return not self.ssl_verify.lower().strip() == 'no'
1066 # initialize pkg list to install
1068 self.__sanity_check()
1070 self._required_pkgs = \
1071 kickstart.get_packages(self.ks, self._get_required_packages())
1072 self._excluded_pkgs = \
1073 kickstart.get_excluded(self.ks, self._get_excluded_packages())
1074 self._required_groups = kickstart.get_groups(self.ks)
1076 self._required_pkgs = None
1077 self._excluded_pkgs = None
1078 self._required_groups = None
1081 repo_urls = self.extrarepos
1083 pkg_manager = self.get_pkg_manager()
1086 if hasattr(self, 'install_pkgs') and self.install_pkgs:
1087 if 'debuginfo' in self.install_pkgs:
1088 pkg_manager.install_debuginfo = True
1090 for repo in kickstart.get_repos(self.ks, repo_urls, self.ignore_ksrepo):
1091 (name, baseurl, mirrorlist, inc, exc,
1092 proxy, proxy_username, proxy_password, debuginfo,
1093 source, gpgkey, disable, ssl_verify, nocache,
1094 cost, priority) = repo
1096 ssl_verify = get_ssl_verify(ssl_verify)
1097 yr = pkg_manager.addRepository(name, baseurl, mirrorlist, proxy,
1098 proxy_username, proxy_password, inc, exc, ssl_verify,
1099 nocache, cost, priority)
1101 if kickstart.exclude_docs(self.ks):
1102 rpm.addMacro("_excludedocs", "1")
1103 rpm.addMacro("_dbpath", "/var/lib/rpm")
1104 rpm.addMacro("__file_context_path", "%{nil}")
1105 if kickstart.inst_langs(self.ks) != None:
1106 rpm.addMacro("_install_langs", kickstart.inst_langs(self.ks))
1109 self.__preinstall_packages(pkg_manager)
1110 self.__select_packages(pkg_manager)
1111 self.__select_groups(pkg_manager)
1112 self.__deselect_packages(pkg_manager)
1113 #self.__localinst_packages(pkg_manager)
1114 self.__check_packages(pkg_manager)
1116 BOOT_SAFEGUARD = 256L * 1024 * 1024 # 256M
1117 checksize = self._root_fs_avail
1119 checksize -= BOOT_SAFEGUARD
1120 if self.target_arch:
1121 pkg_manager._add_prob_flags(rpm.RPMPROB_FILTER_IGNOREARCH)
1123 # If we have multiple partitions, don't check diskspace when rpm run transaction
1124 # because rpm check '/' partition only.
1125 if self.multiple_partitions:
1126 pkg_manager._add_prob_flags(rpm.RPMPROB_FILTER_DISKSPACE)
1127 pkg_manager.runInstall(checksize)
1128 except CreatorError, e:
1130 except KeyboardInterrupt:
1133 self._pkgs_content = pkg_manager.getAllContent()
1134 self._pkgs_license = pkg_manager.getPkgsLicense()
1135 self._pkgs_vcsinfo = pkg_manager.getVcsInfo()
1136 self.__attachment_packages(pkg_manager)
1140 if checkScriptletError(self._instroot + "/tmp/.postscript/error/", "_error"):
1141 showErrorInfo(self._instroot + "/tmp/.preload_install_error")
1142 raise CreatorError('scriptlet errors occurred')
1147 # do some clean up to avoid lvm info leakage. this sucks.
1148 for subdir in ("cache", "backup", "archive"):
1149 lvmdir = self._instroot + "/etc/lvm/" + subdir
1151 for f in os.listdir(lvmdir):
1152 os.unlink(lvmdir + "/" + f)
1156 def postinstall(self):
1159 def _get_sign_scripts_env(self):
1160 """Return an environment dict for %post-umount scripts.
1162 This is the hook where subclasses may specify some environment
1163 variables for %post-umount scripts by return a dict containing the
1164 desired environment.
1169 # Directory path of images
1171 env['IMG_DIR_PATH'] = str(self._imgdir)
1175 for item in self._instloops:
1176 imgfiles.append(item['name'])
1178 imgpaths.append(os.path.join(self._imgdir, item['name']))
1181 env['IMG_FILES'] = ' '.join(imgfiles)
1183 # Absolute path of images
1184 env['IMG_PATHS'] = ' '.join(imgpaths)
1188 def run_sign_scripts(self):
1189 if kickstart.get_sign_scripts(self.ks)==[]:
1191 msger.info("Running sign scripts ...")
1192 if os.path.exists(self._instroot + "/tmp"):
1193 shutil.rmtree(self._instroot + "/tmp")
1194 os.mkdir (self._instroot + "/tmp", 0755)
1195 for s in kickstart.get_sign_scripts(self.ks):
1196 (fd, path) = tempfile.mkstemp(prefix = "ks-runscript-",
1197 dir = self._instroot + "/tmp")
1198 s.script = s.script.replace("\r", "")
1199 os.write(fd, s.script)
1200 if s.interp == '/bin/sh' or s.interp == '/bin/bash':
1202 os.write(fd, 'exit 0\n')
1204 os.chmod(path, 0700)
1206 for item in os.listdir(self._imgdir):
1207 sub = os.path.splitext(item)[1]
1209 shutil.move(os.path.join(self._imgdir, item),
1210 os.path.join(self._instroot + "/tmp", item))
1211 oldoutdir = os.getcwd()
1212 os.chdir(self._instroot + "/tmp")
1214 env = self._get_sign_scripts_env()
1215 #*.img files are moved to self._instroot + "/tmp" directory in running runscripts
1216 env['IMG_PATHS'] = env['IMG_PATHS'].replace(self._imgdir,self._instroot + "/tmp")
1219 p = subprocess.Popen([s.interp, path],
1221 stdout = subprocess.PIPE,
1222 stderr = subprocess.STDOUT)
1223 while p.poll() == None:
1224 msger.info(p.stdout.readline().strip())
1225 if p.returncode != 0:
1226 raise CreatorError("Failed to execute %%sign script "
1227 "with '%s'" % (s.interp))
1228 except OSError, (err, msg):
1229 raise CreatorError("Failed to execute %%sign script "
1230 "with '%s' : %s" % (s.interp, msg))
1234 for item in os.listdir(self._instroot + "/tmp"):
1235 shutil.move(os.path.join(self._instroot + "/tmp", item),
1236 os.path.join(self._imgdir, item))
1238 def __run_post_scripts(self):
1239 msger.info("Running post scripts ...")
1240 if os.path.exists(self._instroot + "/tmp"):
1241 shutil.rmtree(self._instroot + "/tmp")
1242 os.mkdir (self._instroot + "/tmp", 0755)
1243 for s in kickstart.get_post_scripts(self.ks):
1244 (fd, path) = tempfile.mkstemp(prefix = "ks-postscript-",
1245 dir = self._instroot + "/tmp")
1247 s.script = s.script.replace("\r", "")
1248 os.write(fd, s.script)
1249 if s.interp == '/bin/sh' or s.interp == '/bin/bash':
1251 os.write(fd, 'exit 0\n')
1253 os.chmod(path, 0700)
1255 env = self._get_post_scripts_env(s.inChroot)
1256 if 'PATH' not in env:
1257 env['PATH'] = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin'
1263 preexec = self._chroot
1264 script = "/tmp/" + os.path.basename(path)
1266 start_time = time.time()
1269 p = subprocess.Popen([s.interp, script],
1270 preexec_fn = preexec,
1272 stdout = subprocess.PIPE,
1273 stderr = subprocess.STDOUT)
1274 while p.poll() == None:
1275 msger.info(p.stdout.readline().strip())
1276 end_time = time.time()
1277 if (end_time - start_time)/60 > MAX_RUN_TIME:
1278 raise CreatorError("Your post script is executed more than "+MAX_RUN_TIME+"mins, please check it!")
1279 if p.returncode != 0:
1280 raise CreatorError("Failed to execute %%post script "
1281 "with '%s'" % (s.interp))
1282 except OSError, (err, msg):
1283 raise CreatorError("Failed to execute %%post script "
1284 "with '%s' : %s" % (s.interp, msg))
1288 def __save_repo_keys(self, repodata):
1292 gpgkeydir = "/etc/pki/rpm-gpg"
1293 fs.makedirs(self._instroot + gpgkeydir)
1294 for repo in repodata:
1296 repokey = gpgkeydir + "/RPM-GPG-KEY-%s" % repo["name"]
1297 shutil.copy(repo["repokey"], self._instroot + repokey)
1299 def configure(self, repodata = None):
1300 """Configure the system image according to the kickstart.
1302 This method applies the (e.g. keyboard or network) configuration
1303 specified in the kickstart and executes the kickstart %post scripts.
1305 If necessary, it also prepares the image to be bootable by e.g.
1306 creating an initrd and bootloader configuration.
1309 ksh = self.ks.handler
1311 msger.info('Applying configurations ...')
1313 kickstart.LanguageConfig(self._instroot).apply(ksh.lang)
1314 kickstart.KeyboardConfig(self._instroot).apply(ksh.keyboard)
1315 kickstart.TimezoneConfig(self._instroot).apply(ksh.timezone)
1316 #kickstart.AuthConfig(self._instroot).apply(ksh.authconfig)
1317 kickstart.FirewallConfig(self._instroot).apply(ksh.firewall)
1318 kickstart.RootPasswordConfig(self._instroot).apply(ksh.rootpw)
1319 kickstart.UserConfig(self._instroot).apply(ksh.user)
1320 kickstart.ServicesConfig(self._instroot).apply(ksh.services)
1321 kickstart.XConfig(self._instroot).apply(ksh.xconfig)
1322 kickstart.NetworkConfig(self._instroot).apply(ksh.network)
1323 kickstart.RPMMacroConfig(self._instroot).apply(self.ks)
1324 kickstart.DesktopConfig(self._instroot).apply(ksh.desktop)
1325 self.__save_repo_keys(repodata)
1326 kickstart.MoblinRepoConfig(self._instroot).apply(ksh.repo, repodata, self.repourl)
1328 msger.warning("Failed to apply configuration to image")
1331 self._create_bootconfig()
1332 self.__run_post_scripts()
1334 def launch_shell(self, launch):
1335 """Launch a shell in the install root.
1337 This method is launches a bash shell chroot()ed in the install root;
1338 this can be useful for debugging.
1342 msger.info("Launching shell. Exit to continue.")
1343 subprocess.call(["/bin/bash"], preexec_fn = self._chroot)
1345 def do_genchecksum(self, image_name):
1346 if not self._genchecksum:
1349 md5sum = misc.get_md5sum(image_name)
1350 with open(image_name + ".md5sum", "w") as f:
1351 f.write("%s %s" % (md5sum, os.path.basename(image_name)))
1352 self.outimage.append(image_name+".md5sum")
1354 def remove_exclude_image(self):
1355 for item in self._instloops[:]:
1356 if item['exclude_image']:
1357 msger.info("Removing %s in image." % item['name'])
1358 imgfile = os.path.join(self._imgdir, item['name'])
1361 except OSError as err:
1362 if err.errno == errno.ENOENT:
1364 self._instloops.remove(item)
1366 def create_cpio_image(self):
1367 for item in self._instloops:
1368 if item['cpioopts']:
1369 msger.info("Create image by cpio.")
1370 tmp_cpio = self.__builddir + "/tmp-cpio"
1371 if not os.path.exists(tmp_cpio):
1373 tmp_cpio_imgfile = os.path.join(tmp_cpio, item['name'])
1375 cpiocmd = fs.find_binary_path('cpio')
1377 oldoutdir = os.getcwd()
1378 os.chdir(os.path.join(self._instroot, item['mountpoint'].lstrip('/')))
1379 # find . | cpio --create --'format=newc' | gzip > ../ramdisk.img
1380 runner.show('find . | cpio --create %s | gzip > %s' % (item['cpioopts'], tmp_cpio_imgfile))
1382 except OSError, (errno, msg):
1383 raise CreatorError("Create image by cpio error: %s" % msg)
1385 def copy_cpio_image(self):
1386 for item in self._instloops:
1387 if item['cpioopts']:
1388 tmp_cpio = self.__builddir + "/tmp-cpio"
1389 msger.info("Copy cpio image from %s to %s." %(tmp_cpio, self._imgdir))
1391 shutil.copyfile(os.path.join(tmp_cpio, item['name']),os.path.join(self._imgdir, item['name']))
1393 raise CreatorError("Copy cpio image error")
1394 os.remove(os.path.join(tmp_cpio, item['name']))
1395 if not os.listdir(tmp_cpio):
1396 shutil.rmtree(tmp_cpio, ignore_errors=True)
1398 def package(self, destdir = "."):
1399 """Prepares the created image for final delivery.
1401 In its simplest form, this method merely copies the install root to the
1402 supplied destination directory; other subclasses may choose to package
1403 the image by e.g. creating a bootable ISO containing the image and
1404 bootloader configuration.
1406 destdir -- the directory into which the final image should be moved;
1407 this defaults to the current directory.
1410 self.remove_exclude_image()
1412 self._stage_final_image()
1414 if not os.path.exists(destdir):
1415 fs.makedirs(destdir)
1417 if self._recording_pkgs:
1418 self._save_recording_pkgs(destdir)
1420 # For image formats with two or multiple image files, it will be
1421 # better to put them under a directory
1422 if self.image_format in ("raw", "vmdk", "vdi", "nand", "mrstnand"):
1423 destdir = os.path.join(destdir, "%s-%s" \
1424 % (self.name, self.image_format))
1425 msger.debug("creating destination dir: %s" % destdir)
1426 fs.makedirs(destdir)
1428 # Ensure all data is flushed to _outdir
1429 runner.quiet('sync')
1431 misc.check_space_pre_cp(self._outdir, destdir)
1432 for f in os.listdir(self._outdir):
1433 shutil.move(os.path.join(self._outdir, f),
1434 os.path.join(destdir, f))
1435 self.outimage.append(os.path.join(destdir, f))
1436 self.do_genchecksum(os.path.join(destdir, f))
1438 def print_outimage_info(self):
1439 msg = "The new image can be found here:\n"
1440 self.outimage.sort()
1441 for file in self.outimage:
1442 msg += ' %s\n' % os.path.abspath(file)
1446 def check_depend_tools(self):
1447 for tool in self._dep_checks:
1448 fs.find_binary_path(tool)
1450 def package_output(self, image_format, destdir = ".", package="none"):
1451 if not package or package == "none":
1454 destdir = os.path.abspath(os.path.expanduser(destdir))
1455 (pkg, comp) = os.path.splitext(package)
1457 comp=comp.lstrip(".")
1461 dst = "%s/%s-%s.tar.%s" %\
1462 (destdir, self.name, image_format, comp)
1464 dst = "%s/%s-%s.tar" %\
1465 (destdir, self.name, image_format)
1467 msger.info("creating %s" % dst)
1468 tar = tarfile.open(dst, "w:" + comp)
1470 for file in self.outimage:
1471 msger.info("adding %s to %s" % (file, dst))
1473 arcname=os.path.join("%s-%s" \
1474 % (self.name, image_format),
1475 os.path.basename(file)))
1476 if os.path.isdir(file):
1477 shutil.rmtree(file, ignore_errors = True)
1483 '''All the file in outimage has been packaged into tar.* file'''
1484 self.outimage = [dst]
1486 def release_output(self, config, destdir, release):
1487 """ Create release directory and files
1491 """ release path """
1492 return os.path.join(destdir, fn)
1494 outimages = self.outimage
1497 new_kspath = _rpath(self.name+'.ks')
1498 with open(config) as fr:
1499 with open(new_kspath, "w") as wf:
1500 # When building a release we want to make sure the .ks
1501 # file generates the same build even when --release not used.
1502 wf.write(fr.read().replace("@BUILD_ID@", release))
1503 outimages.append(new_kspath)
1505 # save log file, logfile is only available in creator attrs
1506 if hasattr(self, 'releaselog') and self.releaselog:
1507 outimages.append(self.logfile)
1509 # rename iso and usbimg
1510 for f in os.listdir(destdir):
1511 if f.endswith(".iso"):
1512 newf = f[:-4] + '.img'
1513 elif f.endswith(".usbimg"):
1514 newf = f[:-7] + '.img'
1517 os.rename(_rpath(f), _rpath(newf))
1518 outimages.append(_rpath(newf))
1520 # generate MD5SUMS SHA1SUMS SHA256SUMS
1521 def generate_hashsum(hash_name, hash_method):
1522 with open(_rpath(hash_name), "w") as wf:
1523 for f in os.listdir(destdir):
1524 if f.endswith('SUMS'):
1527 if os.path.isdir(os.path.join(destdir, f)):
1530 hash_value = hash_method(_rpath(f))
1531 # There needs to be two spaces between the sum and
1532 # filepath to match the syntax with md5sum,sha1sum,
1533 # sha256sum. This way also *sum -c *SUMS can be used.
1534 wf.write("%s %s\n" % (hash_value, f))
1536 outimages.append("%s/%s" % (destdir, hash_name))
1539 'MD5SUMS' : misc.get_md5sum,
1540 'SHA1SUMS' : misc.get_sha1sum,
1541 'SHA256SUMS' : misc.get_sha256sum
1544 for k, v in hash_dict.items():
1545 generate_hashsum(k, v)
1547 # Filter out the nonexist file
1548 for fp in outimages[:]:
1549 if not os.path.exists("%s" % fp):
1550 outimages.remove(fp)
1552 def copy_kernel(self):
1553 """ Copy kernel files to the outimage directory.
1554 NOTE: This needs to be called before unmounting the instroot.
1557 if not self._need_copy_kernel:
1560 if not os.path.exists(self.destdir):
1561 os.makedirs(self.destdir)
1563 for kernel in glob.glob("%s/boot/vmlinuz-*" % self._instroot):
1564 kernelfilename = "%s/%s-%s" % (self.destdir,
1566 os.path.basename(kernel))
1567 msger.info('copy kernel file %s as %s' % (os.path.basename(kernel),
1569 shutil.copy(kernel, kernelfilename)
1570 self.outimage.append(kernelfilename)
1572 def copy_attachment(self):
1573 """ Subclass implement it to handle attachment files
1574 NOTE: This needs to be called before unmounting the instroot.
1578 def get_pkg_manager(self):
1579 return self.pkgmgr(target_arch = self.target_arch,
1580 instroot = self._instroot,
1581 cachedir = self.cachedir,
1582 strict_mode = self.strict_mode)
1584 def create_manifest(self):
1585 def get_pack_suffix():
1586 return '.' + self.pack_to.split('.', 1)[1]
1588 if not os.path.exists(self.destdir):
1589 os.makedirs(self.destdir)
1591 now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
1592 manifest_dict = {'version': VERSION,
1595 manifest_dict.update({'format': self.img_format})
1597 if hasattr(self, 'logfile') and self.logfile:
1598 manifest_dict.update({'log_file': self.logfile})
1600 if self.image_files:
1602 self.image_files.update({'pack': get_pack_suffix()})
1603 manifest_dict.update({self.img_format: self.image_files})
1605 msger.info('Creating manifest file...')
1606 manifest_file_path = os.path.join(self.destdir, 'manifest.json')
1607 with open(manifest_file_path, 'w') as fest_file:
1608 json.dump(manifest_dict, fest_file, indent=4)
1609 self.outimage.append(manifest_file_path)