import re
import tarfile
import glob
+import json
+from datetime import datetime
import rpm
-
+import time
from mic import kickstart
-from mic import msger
+from mic import msger, __version__ as VERSION
from mic.utils.errors import CreatorError, Abort
from mic.utils import misc, grabber, runner, fs_related as fs
+from mic.chroot import kill_proc_inchroot
+from mic.archive import get_archive_suffixes
+#post script max run time
+MAX_RUN_TIME = 120
class BaseImageCreator(object):
"""Installs a system to a chroot directory.
imgcreate.ImageCreator(ks, "foo").create()
"""
+ # Output image format
+ img_format = ''
def __del__(self):
self.cleanup()
"""
self.pkgmgr = pkgmgr
+ self.distro_name = ""
self.__builddir = None
self.__bindmounts = []
self.name = "target"
self.tmpdir = "/var/tmp/mic"
self.cachedir = "/var/tmp/mic/cache"
+ self.workdir = "/var/tmp/mic/build"
self.destdir = "."
+ self.installerfw_prefix = "INSTALLERFW_"
self.target_arch = "noarch"
+ self.strict_mode = False
self._local_pkgs_path = None
self.pack_to = None
self.repourl = {}
+ self.multiple_partitions = False
# If the kernel is save to the destdir when copy_kernel cmd is called.
self._need_copy_kernel = False
+ # setup tmpfs tmpdir when enabletmpfs is True
+ self.enabletmpfs = False
if createopts:
# Mapping table for variables that have different names.
optmap = {"pkgmgr" : "pkgmgr_name",
- "outdir" : "destdir",
"arch" : "target_arch",
"local_pkgs_path" : "_local_pkgs_path",
"copy_kernel" : "_need_copy_kernel",
+ "strict_mode" : "strict_mode",
}
# update setting from createopts
if '@NAME@' in self.pack_to:
self.pack_to = self.pack_to.replace('@NAME@', self.name)
(tar, ext) = os.path.splitext(self.pack_to)
- if ext in (".gz", ".bz2") and tar.endswith(".tar"):
+ if ext in (".gz", ".bz2", ".lzo", ".bz") and tar.endswith(".tar"):
ext = ".tar" + ext
- if ext not in misc.pack_formats:
+ if ext not in get_archive_suffixes():
self.pack_to += ".tar"
self._dep_checks = ["ls", "bash", "cp", "echo", "modprobe"]
# Output image file names
self.outimage = []
-
+ # Output info related with manifest
+ self.image_files = {}
# A flag to generate checksum
self._genchecksum = False
if part.fstype and part.fstype == "btrfs":
self._dep_checks.append("mkfs.btrfs")
break
-
- if self.target_arch and self.target_arch.startswith("arm"):
- for dep in self._dep_checks:
- if dep == "extlinux":
- self._dep_checks.remove(dep)
-
- if not os.path.exists("/usr/bin/qemu-arm") or \
- not misc.is_statically_linked("/usr/bin/qemu-arm"):
- self._dep_checks.append("qemu-arm-static")
-
- if os.path.exists("/proc/sys/vm/vdso_enabled"):
- vdso_fh = open("/proc/sys/vm/vdso_enabled","r")
- vdso_value = vdso_fh.read().strip()
- vdso_fh.close()
- if (int)(vdso_value) == 1:
- msger.warning("vdso is enabled on your host, which might "
- "cause problems with arm emulations.\n"
- "\tYou can disable vdso with following command before "
- "starting image build:\n"
- "\techo 0 | sudo tee /proc/sys/vm/vdso_enabled")
+ if part.fstype == "cpio":
+ part.fstype = "ext4"
+ if len(self.ks.handler.partition.partitions) > 1:
+ self.multiple_partitions = True
+
+ if self.target_arch:
+ if self.target_arch.startswith("arm"):
+ for dep in self._dep_checks:
+ if dep == "extlinux":
+ self._dep_checks.remove(dep)
+
+ if not os.path.exists("/usr/bin/qemu-arm") or \
+ not misc.is_statically_linked("/usr/bin/qemu-arm"):
+ self._dep_checks.append("qemu-arm-static")
+
+ if os.path.exists("/proc/sys/vm/vdso_enabled"):
+ vdso_fh = open("/proc/sys/vm/vdso_enabled","r")
+ vdso_value = vdso_fh.read().strip()
+ vdso_fh.close()
+ if (int)(vdso_value) == 1:
+ msger.warning("vdso is enabled on your host, which might "
+ "cause problems with arm emulations.\n"
+ "\tYou can disable vdso with following command before "
+ "starting image build:\n"
+ "\techo 0 | sudo tee /proc/sys/vm/vdso_enabled")
+ elif self.target_arch == "mipsel":
+ for dep in self._dep_checks:
+ if dep == "extlinux":
+ self._dep_checks.remove(dep)
+
+ if not os.path.exists("/usr/bin/qemu-mipsel") or \
+ not misc.is_statically_linked("/usr/bin/qemu-mipsel"):
+ self._dep_checks.append("qemu-mipsel-static")
+
+ if os.path.exists("/proc/sys/vm/vdso_enabled"):
+ vdso_fh = open("/proc/sys/vm/vdso_enabled","r")
+ vdso_value = vdso_fh.read().strip()
+ vdso_fh.close()
+ if (int)(vdso_value) == 1:
+ msger.warning("vdso is enabled on your host, which might "
+ "cause problems with mipsel emulations.\n"
+ "\tYou can disable vdso with following command before "
+ "starting image build:\n"
+ "\techo 0 | sudo tee /proc/sys/vm/vdso_enabled")
# make sure the specified tmpdir and cachedir exist
if not os.path.exists(self.tmpdir):
if not os.path.exists(destdir):
os.makedirs(destdir)
- if 'name' in self._recording_pkgs :
+
+ content = None
+ if 'vcs' in self._recording_pkgs:
+ vcslst = ["%s %s" % (k, v) for (k, v) in self._pkgs_vcsinfo.items()]
+ content = '\n'.join(sorted(vcslst))
+ elif 'name' in self._recording_pkgs:
+ content = '\n'.join(pkgs)
+ if content:
namefile = os.path.join(destdir, self.name + '.packages')
f = open(namefile, "w")
- content = '\n'.join(pkgs)
f.write(content)
f.close()
self.outimage.append(namefile);
# if 'content', save more details
- if 'content' in self._recording_pkgs :
+ if 'content' in self._recording_pkgs:
contfile = os.path.join(destdir, self.name + '.files')
f = open(contfile, "w")
f.close()
self.outimage.append(licensefile)
- if 'vcs' in self._recording_pkgs:
- vcsfile = os.path.join(destdir, self.name + '.vcs')
- f = open(vcsfile, "w")
- f.write('\n'.join(["%s\n %s" % (k, v)
- for (k, v) in self._pkgs_vcsinfo.items()]))
- f.close()
-
def _get_required_packages(self):
"""Return a list of required packages.
s += "sysfs /sys sysfs defaults 0 0\n"
return s
+ def _set_part_env(self, pnum, prop, value):
+ """ This is a helper function which generates an environment variable
+ for a property "prop" with value "value" of a partition number "pnum".
+
+ The naming convention is:
+ * Variables start with INSTALLERFW_PART
+ * Then goes the partition number, the order is the same as
+ specified in the KS file
+ * Then goes the property name
+ """
+
+ if value == None:
+ value = ""
+ else:
+ value = str(value)
+
+ name = self.installerfw_prefix + ("PART%d_" % pnum) + prop
+ return { name : value }
+
def _get_post_scripts_env(self, in_chroot):
"""Return an environment dict for %post scripts.
variables for %post scripts by return a dict containing the desired
environment.
- By default, this returns an empty dict.
-
in_chroot -- whether this %post script is to be executed chroot()ed
into _instroot.
-
"""
- return {}
+
+ env = {}
+ pnum = 0
+
+ for p in kickstart.get_partitions(self.ks):
+ env.update(self._set_part_env(pnum, "SIZE", p.size))
+ env.update(self._set_part_env(pnum, "MOUNTPOINT", p.mountpoint))
+ env.update(self._set_part_env(pnum, "FSTYPE", p.fstype))
+ env.update(self._set_part_env(pnum, "LABEL", p.label))
+ env.update(self._set_part_env(pnum, "FSOPTS", p.fsopts))
+ env.update(self._set_part_env(pnum, "BOOTFLAG", p.active))
+ env.update(self._set_part_env(pnum, "ALIGN", p.align))
+ env.update(self._set_part_env(pnum, "TYPE_ID", p.part_type))
+ env.update(self._set_part_env(pnum, "UUID", p.uuid))
+ env.update(self._set_part_env(pnum, "DEVNODE",
+ "/dev/%s%d" % (p.disk, pnum + 1)))
+ env.update(self._set_part_env(pnum, "DISK_DEVNODE",
+ "/dev/%s" % p.disk))
+ pnum += 1
+
+ # Count of paritions
+ env[self.installerfw_prefix + "PART_COUNT"] = str(pnum)
+
+ # Partition table format
+ ptable_format = self.ks.handler.bootloader.ptable
+ env[self.installerfw_prefix + "PTABLE_FORMAT"] = ptable_format
+
+ # The kerned boot parameters
+ kernel_opts = self.ks.handler.bootloader.appendLine
+ env[self.installerfw_prefix + "KERNEL_OPTS"] = kernel_opts
+
+ # Name of the image creation tool
+ env[self.installerfw_prefix + "INSTALLER_NAME"] = "mic"
+
+ # The real current location of the mounted file-systems
+ if in_chroot:
+ mount_prefix = "/"
+ else:
+ mount_prefix = self._instroot
+ env[self.installerfw_prefix + "MOUNT_PREFIX"] = mount_prefix
+
+ # These are historical variables which lack the common name prefix
+ if not in_chroot:
+ env["INSTALL_ROOT"] = self._instroot
+ env["IMG_NAME"] = self._name
+
+ return env
def __get_imgname(self):
return self.name
return
try:
- self.__builddir = tempfile.mkdtemp(dir = self.tmpdir,
+ self.workdir = os.path.join(self.tmpdir, "build")
+ if not os.path.exists(self.workdir):
+ os.makedirs(self.workdir)
+ self.__builddir = tempfile.mkdtemp(dir = self.workdir,
prefix = "imgcreate-")
except OSError, (err, msg):
raise CreatorError("Failed create build directory in %s: %s" %
return self.cachedir
def __sanity_check(self):
- """Ensure that the config we've been given is sane."""
+ """Ensure that the config we've been given is same."""
if not (kickstart.get_packages(self.ks) or
kickstart.get_groups(self.ks)):
raise CreatorError("No packages or groups specified")
raise CreatorError("No repositories specified")
def __write_fstab(self):
- fstab = open(self._instroot + "/etc/fstab", "w")
- fstab.write(self._get_fstab())
- fstab.close()
+ if kickstart.use_installerfw(self.ks, "fstab"):
+ # The fstab file will be generated by installer framework scripts
+ # instead.
+ return None
+ fstab_contents = self._get_fstab()
+ if fstab_contents:
+ fstab = open(self._instroot + "/etc/fstab", "w")
+ fstab.write(fstab_contents)
+ fstab.close()
def __create_minimal_dev(self):
"""Create a minimal /dev so that we don't corrupt the host /dev"""
os.umask(origumask)
+ def __setup_tmpdir(self):
+ if not self.enabletmpfs:
+ return
+
+ runner.show('mount -t tmpfs -o size=4G tmpfs %s' % self.workdir)
+
+ def __clean_tmpdir(self):
+ if not self.enabletmpfs:
+ return
+
+ runner.show('umount -l %s' % self.workdir)
+
def mount(self, base_on = None, cachedir = None):
"""Setup the target filesystem in preparation for an install.
multiple installs.
"""
+ self.__setup_tmpdir()
self.__ensure_builddir()
# prevent popup dialog in Ubuntu(s)
"/usr/bin"):
fs.makedirs(self._instroot + d)
- if self.target_arch and self.target_arch.startswith("arm"):
+ if self.target_arch and self.target_arch.startswith("arm") or \
+ self.target_arch == "aarch64" or self.target_arch == "mipsel" :
self.qemu_emulator = misc.setup_qemu_emulator(self._instroot,
self.target_arch)
# reset settings of popup dialog in Ubuntu(s)
misc.unhide_loopdev_presentation()
+
def cleanup(self):
"""Unmounts the target filesystem and deletes temporary files.
if not self.__builddir:
return
+ kill_proc_inchroot(self._instroot)
+
self.unmount()
shutil.rmtree(self.__builddir, ignore_errors = True)
self.__builddir = None
+ self.__clean_tmpdir()
+
def __is_excluded_pkg(self, pkg):
if pkg in self._excluded_pkgs:
self._excluded_pkgs.remove(pkg)
for pkg in self._preinstall_pkgs:
pkg_manager.preInstall(pkg)
+ def __check_packages(self, pkg_manager):
+ for pkg in self.check_pkgs:
+ pkg_manager.checkPackage(pkg)
+
def __attachment_packages(self, pkg_manager):
if not self.ks:
return
if not os.path.exists(fpath):
# download pkgs
try:
- fpath = grabber.myurlgrab(url, fpath, proxies, None)
+ fpath = grabber.myurlgrab(url.full, fpath, proxies, None)
except CreatorError:
raise
fpath = os.path.join(root, fname)
self._attachment.append(fpath)
- def install(self, repo_urls = {}):
+ def install(self, repo_urls=None):
"""Install packages into the install root.
This function installs the packages listed in the supplied kickstart
into the install root. By default, the packages are installed from the
repository URLs specified in the kickstart.
- repo_urls -- a dict which maps a repository name to a repository URL;
+ repo_urls -- a dict which maps a repository name to a repository;
if supplied, this causes any repository URLs specified in
the kickstart to be overridden.
"""
+ def checkScriptletError(dirname, suffix):
+ if os.path.exists(dirname):
+ list = os.listdir(dirname)
+ for line in list:
+ filepath = os.path.join(dirname, line)
+ if os.path.isfile(filepath) and 0 < line.find(suffix):
+ return True
+ else:
+ continue
+
+ return False
+
+ def showErrorInfo(filepath):
+ if os.path.isfile(filepath):
+ for line in open(filepath):
+ msger.info("The error install package info: %s" % line)
+ else:
+ msger.info("%s is not found." % filepath)
+
+ def get_ssl_verify(ssl_verify=None):
+ if ssl_verify is not None:
+ return not ssl_verify.lower().strip() == 'no'
+ else:
+ return not self.ssl_verify.lower().strip() == 'no'
# initialize pkg list to install
if self.ks:
self._excluded_pkgs = None
self._required_groups = None
+ if not repo_urls:
+ repo_urls = self.extrarepos
+
pkg_manager = self.get_pkg_manager()
pkg_manager.setup()
if 'debuginfo' in self.install_pkgs:
pkg_manager.install_debuginfo = True
- for repo in kickstart.get_repos(self.ks, repo_urls):
+ for repo in kickstart.get_repos(self.ks, repo_urls, self.ignore_ksrepo):
(name, baseurl, mirrorlist, inc, exc,
proxy, proxy_username, proxy_password, debuginfo,
source, gpgkey, disable, ssl_verify, nocache,
cost, priority) = repo
+ ssl_verify = get_ssl_verify(ssl_verify)
yr = pkg_manager.addRepository(name, baseurl, mirrorlist, proxy,
proxy_username, proxy_password, inc, exc, ssl_verify,
nocache, cost, priority)
self.__select_groups(pkg_manager)
self.__deselect_packages(pkg_manager)
self.__localinst_packages(pkg_manager)
+ self.__check_packages(pkg_manager)
BOOT_SAFEGUARD = 256L * 1024 * 1024 # 256M
checksize = self._root_fs_avail
checksize -= BOOT_SAFEGUARD
if self.target_arch:
pkg_manager._add_prob_flags(rpm.RPMPROB_FILTER_IGNOREARCH)
+
+ # If we have multiple partitions, don't check diskspace when rpm run transaction
+ # because rpm check '/' partition only.
+ if self.multiple_partitions:
+ pkg_manager._add_prob_flags(rpm.RPMPROB_FILTER_DISKSPACE)
pkg_manager.runInstall(checksize)
except CreatorError, e:
raise
finally:
pkg_manager.close()
+ if checkScriptletError(self._instroot + "/tmp/.postscript/error/", "_error"):
+ showErrorInfo(self._instroot + "/tmp/.preload_install_error")
+ raise CreatorError('scriptlet errors occurred')
+
# hook post install
self.postinstall()
os.chmod(path, 0700)
env = self._get_post_scripts_env(s.inChroot)
+ if 'PATH' not in env:
+ env['PATH'] = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin'
if not s.inChroot:
- env["INSTALL_ROOT"] = self._instroot
- env["IMG_NAME"] = self._name
preexec = None
script = path
else:
preexec = self._chroot
script = "/tmp/" + os.path.basename(path)
+ start_time = time.time()
try:
try:
- subprocess.call([s.interp, script],
- preexec_fn = preexec,
- env = env,
- stdout = sys.stdout,
- stderr = sys.stderr)
+ p = subprocess.Popen([s.interp, script],
+ preexec_fn = preexec,
+ env = env,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.STDOUT)
+ while p.poll() == None:
+ msger.info(p.stdout.readline().strip())
+ end_time = time.time()
+ if (end_time - start_time)/60 > MAX_RUN_TIME:
+ raise CreatorError("Your post script is executed more than "+MAX_RUN_TIME+"mins, please check it!")
except OSError, (err, msg):
raise CreatorError("Failed to execute %%post script "
"with '%s' : %s" % (s.interp, msg))
md5sum = misc.get_md5sum(image_name)
with open(image_name + ".md5sum", "w") as f:
- f.write("%s %s" % (md5sum, os.path.basename(image_name)))
+ f.write("%s %s" % (md5sum, os.path.basename(image_name)))
self.outimage.append(image_name+".md5sum")
+ def remove_exclude_image(self):
+ for item in self._instloops[:]:
+ if item['exclude_image']:
+ msger.info("Removing %s in image." % item['name'])
+ imgfile = os.path.join(self._imgdir, item['name'])
+ try:
+ os.remove(imgfile)
+ except OSError as err:
+ if err.errno == errno.ENOENT:
+ pass
+ self._instloops.remove(item)
+
+ def create_cpio_image(self):
+ for item in self._instloops:
+ if item['cpioopts']:
+ msger.info("Create image by cpio.")
+ tmp_cpio = self.__builddir + "/tmp-cpio"
+ if not os.path.exists(tmp_cpio):
+ os.mkdir(tmp_cpio)
+ tmp_cpio_imgfile = os.path.join(tmp_cpio, item['name'])
+ try:
+ cpiocmd = fs.find_binary_path('cpio')
+ if cpiocmd:
+ oldoutdir = os.getcwd()
+ os.chdir(os.path.join(self._instroot, item['mountpoint'].lstrip('/')))
+ # find . | cpio --create --'format=newc' | gzip > ../ramdisk.img
+ runner.show('find . | cpio --create %s | gzip > %s' % (item['cpioopts'], tmp_cpio_imgfile))
+ os.chdir(oldoutdir)
+ except OSError, (errno, msg):
+ raise errors.CreatorError("Create image by cpio error: %s" % msg)
+
+ def copy_cpio_image(self):
+ for item in self._instloops:
+ if item['cpioopts']:
+ tmp_cpio = self.__builddir + "/tmp-cpio"
+ msger.info("Copy cpio image from %s to %s." %(tmp_cpio, self._imgdir))
+ try:
+ shutil.copyfile(os.path.join(tmp_cpio, item['name']),os.path.join(self._imgdir, item['name']))
+ except IOError:
+ raise errors.CreatorError("Copy cpio image error")
+ os.remove(os.path.join(tmp_cpio, item['name']))
+ if not os.listdir(tmp_cpio):
+ shutil.rmtree(tmp_cpio, ignore_errors=True)
+
def package(self, destdir = "."):
"""Prepares the created image for final delivery.
this defaults to the current directory.
"""
+ self.remove_exclude_image()
+
self._stage_final_image()
if not os.path.exists(destdir):
outimages.append(new_kspath)
# save log file, logfile is only available in creator attrs
- if hasattr(self, 'logfile') and not self.logfile:
- log_path = _rpath(self.name + ".log")
- # touch the log file, else outimages will filter it out
- with open(log_path, 'w') as wf:
- wf.write('')
- msger.set_logfile(log_path)
- outimages.append(_rpath(self.name + ".log"))
+ if hasattr(self, 'releaselog') and self.releaselog:
+ outimages.append(self.logfile)
# rename iso and usbimg
for f in os.listdir(destdir):
os.rename(_rpath(f), _rpath(newf))
outimages.append(_rpath(newf))
- # generate MD5SUMS
- with open(_rpath("MD5SUMS"), "w") as wf:
- for f in os.listdir(destdir):
- if f == "MD5SUMS":
- continue
+ # generate MD5SUMS SHA1SUMS SHA256SUMS
+ def generate_hashsum(hash_name, hash_method):
+ with open(_rpath(hash_name), "w") as wf:
+ for f in os.listdir(destdir):
+ if f.endswith('SUMS'):
+ continue
- if os.path.isdir(os.path.join(destdir, f)):
- continue
+ if os.path.isdir(os.path.join(destdir, f)):
+ continue
+
+ hash_value = hash_method(_rpath(f))
+ # There needs to be two spaces between the sum and
+ # filepath to match the syntax with md5sum,sha1sum,
+ # sha256sum. This way also *sum -c *SUMS can be used.
+ wf.write("%s %s\n" % (hash_value, f))
- md5sum = misc.get_md5sum(_rpath(f))
- # There needs to be two spaces between the sum and
- # filepath to match the syntax with md5sum.
- # This way also md5sum -c MD5SUMS can be used by users
- wf.write("%s *%s\n" % (md5sum, f))
+ outimages.append("%s/%s" % (destdir, hash_name))
- outimages.append("%s/MD5SUMS" % destdir)
+ hash_dict = {
+ 'MD5SUMS' : misc.get_md5sum,
+ 'SHA1SUMS' : misc.get_sha1sum,
+ 'SHA256SUMS' : misc.get_sha256sum
+ }
+
+ for k, v in hash_dict.items():
+ generate_hashsum(k, v)
# Filter out the nonexist file
for fp in outimages[:]:
def get_pkg_manager(self):
return self.pkgmgr(target_arch = self.target_arch,
instroot = self._instroot,
- cachedir = self.cachedir)
+ cachedir = self.cachedir,
+ strict_mode = self.strict_mode)
+
+ def create_manifest(self):
+ def get_pack_suffix():
+ return '.' + self.pack_to.split('.', 1)[1]
+
+ if not os.path.exists(self.destdir):
+ os.makedirs(self.destdir)
+
+ now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+ manifest_dict = {'version': VERSION,
+ 'created': now}
+ if self.img_format:
+ manifest_dict.update({'format': self.img_format})
+
+ if hasattr(self, 'logfile') and self.logfile:
+ manifest_dict.update({'log_file': self.logfile})
+
+ if self.image_files:
+ if self.pack_to:
+ self.image_files.update({'pack': get_pack_suffix()})
+ manifest_dict.update({self.img_format: self.image_files})
+
+ msger.info('Creating manifest file...')
+ manifest_file_path = os.path.join(self.destdir, 'manifest.json')
+ with open(manifest_file_path, 'w') as fest_file:
+ json.dump(manifest_dict, fest_file, indent=4)
+ self.outimage.append(manifest_file_path)
projects / platform / upstream / mic.git / blobdiff