import re
import tarfile
import glob
+import json
+from datetime import datetime
import rpm
-
+import time
from mic import kickstart
-from mic import msger
+from mic import msger, __version__ as VERSION
from mic.utils.errors import CreatorError, Abort
from mic.utils import misc, grabber, runner, fs_related as fs
from mic.chroot import kill_proc_inchroot
+from mic.archive import get_archive_suffixes
+#post script max run time
+MAX_RUN_TIME = 120
class BaseImageCreator(object):
"""Installs a system to a chroot directory.
imgcreate.ImageCreator(ks, "foo").create()
"""
+ # Output image format
+ img_format = ''
def __del__(self):
self.cleanup()
self.destdir = "."
self.installerfw_prefix = "INSTALLERFW_"
self.target_arch = "noarch"
+ self.strict_mode = False
self._local_pkgs_path = None
self.pack_to = None
self.repourl = {}
+ self.multiple_partitions = False
# If the kernel is save to the destdir when copy_kernel cmd is called.
self._need_copy_kernel = False
if createopts:
# Mapping table for variables that have different names.
optmap = {"pkgmgr" : "pkgmgr_name",
- "outdir" : "destdir",
"arch" : "target_arch",
"local_pkgs_path" : "_local_pkgs_path",
"copy_kernel" : "_need_copy_kernel",
+ "strict_mode" : "strict_mode",
}
# update setting from createopts
self.destdir = os.path.abspath(os.path.expanduser(self.destdir))
- if 'release' in createopts and createopts['release']:
- self.name = createopts['release'] + '_' + self.name
-
if self.pack_to:
if '@NAME@' in self.pack_to:
self.pack_to = self.pack_to.replace('@NAME@', self.name)
(tar, ext) = os.path.splitext(self.pack_to)
- if ext in (".gz", ".bz2") and tar.endswith(".tar"):
+ if ext in (".gz", ".bz2", ".lzo", ".bz") and tar.endswith(".tar"):
ext = ".tar" + ext
- if ext not in misc.pack_formats:
+ if ext not in get_archive_suffixes():
self.pack_to += ".tar"
self._dep_checks = ["ls", "bash", "cp", "echo", "modprobe"]
# Output image file names
self.outimage = []
-
+ # Output info related with manifest
+ self.image_files = {}
# A flag to generate checksum
self._genchecksum = False
if part.fstype and part.fstype == "btrfs":
self._dep_checks.append("mkfs.btrfs")
break
-
- if self.target_arch and self.target_arch.startswith("arm"):
- for dep in self._dep_checks:
- if dep == "extlinux":
- self._dep_checks.remove(dep)
-
- if not os.path.exists("/usr/bin/qemu-arm") or \
- not misc.is_statically_linked("/usr/bin/qemu-arm"):
- self._dep_checks.append("qemu-arm-static")
-
- if os.path.exists("/proc/sys/vm/vdso_enabled"):
- vdso_fh = open("/proc/sys/vm/vdso_enabled","r")
- vdso_value = vdso_fh.read().strip()
- vdso_fh.close()
- if (int)(vdso_value) == 1:
- msger.warning("vdso is enabled on your host, which might "
- "cause problems with arm emulations.\n"
- "\tYou can disable vdso with following command before "
- "starting image build:\n"
- "\techo 0 | sudo tee /proc/sys/vm/vdso_enabled")
+ if part.fstype == "cpio":
+ part.fstype = "ext4"
+ if len(self.ks.handler.partition.partitions) > 1:
+ self.multiple_partitions = True
+
+ if self.target_arch:
+ if self.target_arch.startswith("arm"):
+ for dep in self._dep_checks:
+ if dep == "extlinux":
+ self._dep_checks.remove(dep)
+
+ if not os.path.exists("/usr/bin/qemu-arm") or \
+ not misc.is_statically_linked("/usr/bin/qemu-arm"):
+ self._dep_checks.append("qemu-arm-static")
+
+ if os.path.exists("/proc/sys/vm/vdso_enabled"):
+ vdso_fh = open("/proc/sys/vm/vdso_enabled","r")
+ vdso_value = vdso_fh.read().strip()
+ vdso_fh.close()
+ if (int)(vdso_value) == 1:
+ msger.warning("vdso is enabled on your host, which might "
+ "cause problems with arm emulations.\n"
+ "\tYou can disable vdso with following command before "
+ "starting image build:\n"
+ "\techo 0 | sudo tee /proc/sys/vm/vdso_enabled")
+ elif self.target_arch == "mipsel":
+ for dep in self._dep_checks:
+ if dep == "extlinux":
+ self._dep_checks.remove(dep)
+
+ if not os.path.exists("/usr/bin/qemu-mipsel") or \
+ not misc.is_statically_linked("/usr/bin/qemu-mipsel"):
+ self._dep_checks.append("qemu-mipsel-static")
+
+ if os.path.exists("/proc/sys/vm/vdso_enabled"):
+ vdso_fh = open("/proc/sys/vm/vdso_enabled","r")
+ vdso_value = vdso_fh.read().strip()
+ vdso_fh.close()
+ if (int)(vdso_value) == 1:
+ msger.warning("vdso is enabled on your host, which might "
+ "cause problems with mipsel emulations.\n"
+ "\tYou can disable vdso with following command before "
+ "starting image build:\n"
+ "\techo 0 | sudo tee /proc/sys/vm/vdso_enabled")
# make sure the specified tmpdir and cachedir exist
if not os.path.exists(self.tmpdir):
env.update(self._set_part_env(pnum, "MOUNTPOINT", p.mountpoint))
env.update(self._set_part_env(pnum, "FSTYPE", p.fstype))
env.update(self._set_part_env(pnum, "LABEL", p.label))
- env.update(self._set_part_env(pnum, "FSOPTS",
- "defaults,noatime" if not p.fsopts
- else p.fsopts))
+ env.update(self._set_part_env(pnum, "FSOPTS", p.fsopts))
env.update(self._set_part_env(pnum, "BOOTFLAG", p.active))
env.update(self._set_part_env(pnum, "ALIGN", p.align))
env.update(self._set_part_env(pnum, "TYPE_ID", p.part_type))
kernel_opts = self.ks.handler.bootloader.appendLine
env[self.installerfw_prefix + "KERNEL_OPTS"] = kernel_opts
- # Name of the distribution
- env[self.installerfw_prefix + "DISTRO_NAME"] = self.distro_name
-
# Name of the image creation tool
env[self.installerfw_prefix + "INSTALLER_NAME"] = "mic"
return self.cachedir
def __sanity_check(self):
- """Ensure that the config we've been given is sane."""
+ """Ensure that the config we've been given is same."""
if not (kickstart.get_packages(self.ks) or
kickstart.get_groups(self.ks)):
raise CreatorError("No packages or groups specified")
fs.makedirs(self._instroot + d)
if self.target_arch and self.target_arch.startswith("arm") or \
- self.target_arch == "aarch64":
+ self.target_arch == "aarch64" or self.target_arch == "mipsel" :
self.qemu_emulator = misc.setup_qemu_emulator(self._instroot,
self.target_arch)
if not os.path.exists(fpath):
# download pkgs
try:
- fpath = grabber.myurlgrab(url, fpath, proxies, None)
+ fpath = grabber.myurlgrab(url.full, fpath, proxies, None)
except CreatorError:
raise
into the install root. By default, the packages are installed from the
repository URLs specified in the kickstart.
- repo_urls -- a dict which maps a repository name to a repository URL;
+ repo_urls -- a dict which maps a repository name to a repository;
if supplied, this causes any repository URLs specified in
the kickstart to be overridden.
"""
+ def checkScriptletError(dirname, suffix):
+ if os.path.exists(dirname):
+ list = os.listdir(dirname)
+ for line in list:
+ filepath = os.path.join(dirname, line)
+ if os.path.isfile(filepath) and 0 < line.find(suffix):
+ return True
+ else:
+ continue
+
+ return False
+
+ def showErrorInfo(filepath):
+ if os.path.isfile(filepath):
+ for line in open(filepath):
+ msger.info("The error install package info: %s" % line)
+ else:
+ msger.info("%s is not found." % filepath)
+
+ def get_ssl_verify(ssl_verify=None):
+ if ssl_verify is not None:
+ return not ssl_verify.lower().strip() == 'no'
+ else:
+ return not self.ssl_verify.lower().strip() == 'no'
# initialize pkg list to install
if self.ks:
self._excluded_pkgs = None
self._required_groups = None
+ if not repo_urls:
+ repo_urls = self.extrarepos
+
pkg_manager = self.get_pkg_manager()
pkg_manager.setup()
if 'debuginfo' in self.install_pkgs:
pkg_manager.install_debuginfo = True
- for repo in kickstart.get_repos(self.ks, repo_urls):
+ for repo in kickstart.get_repos(self.ks, repo_urls, self.ignore_ksrepo):
(name, baseurl, mirrorlist, inc, exc,
proxy, proxy_username, proxy_password, debuginfo,
source, gpgkey, disable, ssl_verify, nocache,
cost, priority) = repo
+ ssl_verify = get_ssl_verify(ssl_verify)
yr = pkg_manager.addRepository(name, baseurl, mirrorlist, proxy,
proxy_username, proxy_password, inc, exc, ssl_verify,
nocache, cost, priority)
checksize -= BOOT_SAFEGUARD
if self.target_arch:
pkg_manager._add_prob_flags(rpm.RPMPROB_FILTER_IGNOREARCH)
+
+ # If we have multiple partitions, don't check diskspace when rpm run transaction
+ # because rpm check '/' partition only.
+ if self.multiple_partitions:
+ pkg_manager._add_prob_flags(rpm.RPMPROB_FILTER_DISKSPACE)
pkg_manager.runInstall(checksize)
except CreatorError, e:
raise
finally:
pkg_manager.close()
+ if checkScriptletError(self._instroot + "/tmp/.postscript/error/", "_error"):
+ showErrorInfo(self._instroot + "/tmp/.preload_install_error")
+ raise CreatorError('scriptlet errors occurred')
+
# hook post install
self.postinstall()
preexec = self._chroot
script = "/tmp/" + os.path.basename(path)
+ start_time = time.time()
try:
try:
p = subprocess.Popen([s.interp, script],
env = env,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT)
- for entry in p.communicate()[0].splitlines():
- msger.info(entry)
+ while p.poll() == None:
+ msger.info(p.stdout.readline().strip())
+ end_time = time.time()
+ if (end_time - start_time)/60 > MAX_RUN_TIME:
+ raise CreatorError("Your post script is executed more than "+MAX_RUN_TIME+"mins, please check it!")
except OSError, (err, msg):
raise CreatorError("Failed to execute %%post script "
"with '%s' : %s" % (s.interp, msg))
md5sum = misc.get_md5sum(image_name)
with open(image_name + ".md5sum", "w") as f:
- f.write("%s %s" % (md5sum, os.path.basename(image_name)))
+ f.write("%s %s" % (md5sum, os.path.basename(image_name)))
self.outimage.append(image_name+".md5sum")
+ def remove_exclude_image(self):
+ for item in self._instloops[:]:
+ if item['exclude_image']:
+ msger.info("Removing %s in image." % item['name'])
+ imgfile = os.path.join(self._imgdir, item['name'])
+ try:
+ os.remove(imgfile)
+ except OSError as err:
+ if err.errno == errno.ENOENT:
+ pass
+ self._instloops.remove(item)
+
+ def create_cpio_image(self):
+ for item in self._instloops:
+ if item['cpioopts']:
+ msger.info("Create image by cpio.")
+ tmp_cpio = self.__builddir + "/tmp-cpio"
+ if not os.path.exists(tmp_cpio):
+ os.mkdir(tmp_cpio)
+ tmp_cpio_imgfile = os.path.join(tmp_cpio, item['name'])
+ try:
+ cpiocmd = fs.find_binary_path('cpio')
+ if cpiocmd:
+ oldoutdir = os.getcwd()
+ os.chdir(os.path.join(self._instroot, item['mountpoint'].lstrip('/')))
+ # find . | cpio --create --'format=newc' | gzip > ../ramdisk.img
+ runner.show('find . | cpio --create %s | gzip > %s' % (item['cpioopts'], tmp_cpio_imgfile))
+ os.chdir(oldoutdir)
+ except OSError, (errno, msg):
+ raise errors.CreatorError("Create image by cpio error: %s" % msg)
+
+ def copy_cpio_image(self):
+ for item in self._instloops:
+ if item['cpioopts']:
+ tmp_cpio = self.__builddir + "/tmp-cpio"
+ msger.info("Copy cpio image from %s to %s." %(tmp_cpio, self._imgdir))
+ try:
+ shutil.copyfile(os.path.join(tmp_cpio, item['name']),os.path.join(self._imgdir, item['name']))
+ except IOError:
+ raise errors.CreatorError("Copy cpio image error")
+ os.remove(os.path.join(tmp_cpio, item['name']))
+ if not os.listdir(tmp_cpio):
+ shutil.rmtree(tmp_cpio, ignore_errors=True)
+
def package(self, destdir = "."):
"""Prepares the created image for final delivery.
this defaults to the current directory.
"""
+ self.remove_exclude_image()
+
self._stage_final_image()
if not os.path.exists(destdir):
outimages.append(new_kspath)
# save log file, logfile is only available in creator attrs
- if hasattr(self, 'logfile') and not self.logfile:
- log_path = _rpath(self.name + ".log")
- # touch the log file, else outimages will filter it out
- with open(log_path, 'w') as wf:
- wf.write('')
- msger.set_logfile(log_path)
- outimages.append(_rpath(self.name + ".log"))
+ if hasattr(self, 'releaselog') and self.releaselog:
+ outimages.append(self.logfile)
# rename iso and usbimg
for f in os.listdir(destdir):
os.rename(_rpath(f), _rpath(newf))
outimages.append(_rpath(newf))
- # generate MD5SUMS
- with open(_rpath("MD5SUMS"), "w") as wf:
- for f in os.listdir(destdir):
- if f == "MD5SUMS":
- continue
+ # generate MD5SUMS SHA1SUMS SHA256SUMS
+ def generate_hashsum(hash_name, hash_method):
+ with open(_rpath(hash_name), "w") as wf:
+ for f in os.listdir(destdir):
+ if f.endswith('SUMS'):
+ continue
- if os.path.isdir(os.path.join(destdir, f)):
- continue
+ if os.path.isdir(os.path.join(destdir, f)):
+ continue
+
+ hash_value = hash_method(_rpath(f))
+ # There needs to be two spaces between the sum and
+ # filepath to match the syntax with md5sum,sha1sum,
+ # sha256sum. This way also *sum -c *SUMS can be used.
+ wf.write("%s %s\n" % (hash_value, f))
- md5sum = misc.get_md5sum(_rpath(f))
- # There needs to be two spaces between the sum and
- # filepath to match the syntax with md5sum.
- # This way also md5sum -c MD5SUMS can be used by users
- wf.write("%s *%s\n" % (md5sum, f))
+ outimages.append("%s/%s" % (destdir, hash_name))
- outimages.append("%s/MD5SUMS" % destdir)
+ hash_dict = {
+ 'MD5SUMS' : misc.get_md5sum,
+ 'SHA1SUMS' : misc.get_sha1sum,
+ 'SHA256SUMS' : misc.get_sha256sum
+ }
+
+ for k, v in hash_dict.items():
+ generate_hashsum(k, v)
# Filter out the nonexist file
for fp in outimages[:]:
def get_pkg_manager(self):
return self.pkgmgr(target_arch = self.target_arch,
instroot = self._instroot,
- cachedir = self.cachedir)
+ cachedir = self.cachedir,
+ strict_mode = self.strict_mode)
+
+ def create_manifest(self):
+ def get_pack_suffix():
+ return '.' + self.pack_to.split('.', 1)[1]
+
+ if not os.path.exists(self.destdir):
+ os.makedirs(self.destdir)
+
+ now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+ manifest_dict = {'version': VERSION,
+ 'created': now}
+ if self.img_format:
+ manifest_dict.update({'format': self.img_format})
+
+ if hasattr(self, 'logfile') and self.logfile:
+ manifest_dict.update({'log_file': self.logfile})
+
+ if self.image_files:
+ if self.pack_to:
+ self.image_files.update({'pack': get_pack_suffix()})
+ manifest_dict.update({self.img_format: self.image_files})
+
+ msger.info('Creating manifest file...')
+ manifest_file_path = os.path.join(self.destdir, 'manifest.json')
+ with open(manifest_file_path, 'w') as fest_file:
+ json.dump(manifest_dict, fest_file, indent=4)
+ self.outimage.append(manifest_file_path)