From 5182b53225bbb9cbd953f2dd4d37e20fd3b955a1 Mon Sep 17 00:00:00 2001 From: Gui Chen Date: Mon, 19 Nov 2012 19:20:39 +0800 Subject: [PATCH] adapt for new mechanism of bootstrap also fix arm image creation performance issue Signed-off-by: Gui Chen --- distfiles/bootstrap.conf | 39 --------------------------- distfiles/mic.conf.in | 9 +++---- mic/bootstrap.py | 46 +++++++++++++++++++------------ mic/conf.py | 58 +++++++++++++--------------------------- mic/creator.py | 5 ++-- mic/rt_util.py | 30 +++++++-------------- plugins/imager/fs_plugin.py | 2 +- plugins/imager/livecd_plugin.py | 2 +- plugins/imager/liveusb_plugin.py | 2 +- plugins/imager/loop_plugin.py | 2 +- plugins/imager/raw_plugin.py | 2 +- setup.py | 3 +-- 12 files changed, 70 insertions(+), 130 deletions(-) delete mode 100644 distfiles/bootstrap.conf diff --git a/distfiles/bootstrap.conf b/distfiles/bootstrap.conf deleted file mode 100644 index c239eb3..0000000 --- a/distfiles/bootstrap.conf +++ /dev/null @@ -1,39 +0,0 @@ -[main] -distro_name = tizen -rootdir = /var/tmp/mic-bootstrap -enable = true - -[tizen] -optional: systemd syslinux syslinux-extlinux -packages: setup filesystem tzdata libgcc eglibc-common eglibc ncurses-base - ncurses ncurses-libs bash zlib libpython python-base libstdc++ popt - libuuid libblkid libcom_err expat db4 nspr elfutils-libelf sqlite - bzip2-libs bzip2 e2fsprogs-libs libss e2fsprogs libmount libpcre - libfile libxml2 c-ares mtools dosfstools - sed readline libattr libacl coreutils openssl gzip python util-linux - nss-softokn-freebl nss nss-sysinit libidn libcurl curl python-pycurl - xz-libs kmod libcap liblua rpm-libs rpm rpm-python satsolver-tools - mkdevnodes udev libudev libzypp python-zypp mic grep pwdutils lua - device-mapper device-mapper-libs psmisc kpartx parted which gawk - cpio lsof file python-xml python-urlgrabber smack smack-utils - libarchive attr xz qemu-arm-static rpm-security-plugin - expat libxau info libxcb dbus dbus-libs pam - -[meego] -packages: setup filesystem basesystem tzdata meego-release nss-softokn-freebl - ncurses-base ncurses libgcc glibc-common glibc ncurses-libs bash zlib - info popt libstdc++ bzip2-libs chkconfig readline libuuid db4 expat - libcom_err libblkid sqlite file-libs sysvinit-tools shadow-utils passwd - libgpg-error e2fsprogs-libs db4-utils bzip2 libusb pcre gamin glib2 grep - sed libxml2 module-init-tools nspr xz-libs device-mapper-libs libcap - elfutils-libelf liblua hwdata device-mapper kpartx btrfs-progs libgcrypt - MAKEDEV libss e2fsprogs parted v8 usleep libidn cpio findutils mtools - syslinux syslinux-extlinux squashfs-tools procps psmisc libffi mkdevnodes - libksba mingetty libattr libacl coreutils pam openssl libcurl curl gzip - wget sysvinit genisoimage nss-sysinit nss rpm rpm-libs satsolver-tools - logrotate gawk rsync dosfstools gdbm python-libs python python-iniparse - pykickstart m2crypto yum-metadata-parser python-pycurl rpm-python - isomd5sum net-tools eggdbus dbus-glib util-linux-ng ConsoleKit-libs - polkit sysklogd ConsoleKit dbus-libs fastinit dbus udev pacrunner - pacrunner-python python-urlgrabber libudev pth gnupg2 gpgme pygpgme - lua yum libzypp python-zypp mic2 diff --git a/distfiles/mic.conf.in b/distfiles/mic.conf.in index b79603e..3d41eff 100644 --- a/distfiles/mic.conf.in +++ b/distfiles/mic.conf.in @@ -9,7 +9,7 @@ plugin_dir = @PREFIX@/lib/mic/plugins tmpdir= /var/tmp/mic cachedir= /var/tmp/mic/cache outdir= ./mic-output -bootstrapdir= /var/tmp/mic/bootstrap +runtime=bootstrap pkgmgr = auto @@ -29,8 +29,7 @@ pkgmgr = auto [chroot] ; settings for chroot subcommand -[bootstrap1] -name=micbootstrap -bootstrap=http://download.tizen.org/tools/micbootstrap/ -; bootstrap_proxy=http://proxy.yourcompany.com:port +[bootstrap] +rootdir=/var/tmp/mic-bootstrap +packages=mic-bootstrap-x86-arm diff --git a/mic/bootstrap.py b/mic/bootstrap.py index 83ce0bb..c510150 100644 --- a/mic/bootstrap.py +++ b/mic/bootstrap.py @@ -183,37 +183,49 @@ class Bootstrap(object): os.makedirs(self.rootdir) return self.rootdir - def _path(self, pth): - return os.path.join(self.rootdir, pth.lstrip('/')) + def dirsetup(self, rootdir=None): + _path = lambda pth: os.path.join(rootdir, pth.lstrip('/')) - def create(self, repomd, pkglist, optlist=[]): - try: - pkgmgr = MiniBackend(self.get_rootdir()) - pkgmgr.arch = self.arch - pkgmgr.repomd = repomd - pkgmgr.optionals = optlist - map(pkgmgr.selectPackage, pkglist + optlist) - pkgmgr.runInstall() + if not rootdir: + rootdir = self.rootdir - # make /tmp path - tmpdir = self._path('/tmp') + try: + # make /tmp and /etc path + tmpdir = _path('/tmp') if not os.path.exists(tmpdir): os.makedirs(tmpdir) + etcdir = _path('/etc') + if not os.path.exists(etcdir): + os.makedirs(etcdir) # touch distro file - tzdist = self._path('/etc/%s-release' % self.distro) + tzdist = _path('/etc/%s-release' % self.distro) if not os.path.exists(tzdist): with open(tzdist, 'w') as wf: wf.write("bootstrap") + except: + pass + + def create(self, repomd, pkglist, optlist=[]): + try: + pkgmgr = MiniBackend(self.get_rootdir()) + pkgmgr.arch = self.arch + pkgmgr.repomd = repomd + pkgmgr.optionals = optlist + map(pkgmgr.selectPackage, pkglist + optlist) + pkgmgr.runInstall() except (OSError, IOError, errors.CreatorError), err: raise errors.BootstrapError("%s" % err) - def run(self, cmd, chdir, bindmounts=None): + def run(self, cmd, chdir, rootdir=None, bindmounts=None): def mychroot(): - os.chroot(self.rootdir) + os.chroot(rootdir) os.chdir(chdir) + if not rootdir: + rootdir = self.rootdir + if isinstance(cmd, list): shell = False else: @@ -226,14 +238,14 @@ class Bootstrap(object): gloablmounts = None try: proxy.set_proxy_environ() - gloablmounts = setup_chrootenv(self.rootdir, bindmounts) + gloablmounts = setup_chrootenv(rootdir, bindmounts) retcode = subprocess.call(cmd, preexec_fn=mychroot, env=env, shell=shell) except (OSError, IOError), err: raise RuntimeError(err) finally: if self.logfile: msger.log(file(self.logfile).read()) - cleanup_chrootenv(self.rootdir, bindmounts, gloablmounts) + cleanup_chrootenv(rootdir, bindmounts, gloablmounts) proxy.unset_proxy_environ() return retcode diff --git a/mic/conf.py b/mic/conf.py index d0d96ce..153f139 100644 --- a/mic/conf.py +++ b/mic/conf.py @@ -61,6 +61,7 @@ class ConfigMgr(object): "no_proxy": None, "copy_kernel": False, "repourl": {}, + "runtime": "bootstrap", }, 'chroot': { "saveto": None, @@ -69,10 +70,8 @@ class ConfigMgr(object): "shell": False, }, 'bootstrap': { - "enable": False, - "distros": [], # supported distros - "distro_name": None, "rootdir": '/var/tmp/mic-bootstrap', + "packages": [], }, } @@ -94,10 +93,6 @@ class ConfigMgr(object): # initial options from siteconf self._siteconf = siteconf - # set bootstrap from bootstrap.conf - bsconf = os.path.join(os.path.dirname(siteconf), 'bootstrap.conf') - self._parse_bootstrap(bsconf) - if ksconf: self._ksconf = ksconf @@ -146,7 +141,7 @@ class ConfigMgr(object): # append common section items to other sections for section in self.DEFAULTS.keys(): - if section != "common" and not section.startswith('bootstrap'): + if section != "common": getattr(self, section).update(self.common) # check and normalize the scheme of proxy url @@ -163,6 +158,16 @@ class ConfigMgr(object): proxy.set_proxies(self.create['proxy'], self.create['no_proxy']) + # bootstrap option handling + self.set_runtime(self.create['runtime']) + if isinstance(self.bootstrap['packages'], basestring): + packages = self.bootstrap['packages'].replace('\n', ' ') + if packages.find(',') != -1: + packages = packages.split(',') + else: + packages = packages.split() + self.bootstrap['packages'] = packages + def _parse_kickstart(self, ksconf=None): if not ksconf: return @@ -211,37 +216,12 @@ class ConfigMgr(object): misc.selinux_check(self.create['arch'], [p.fstype for p in ks.handler.partition.partitions]) + def set_runtime(self, runtime): + if runtime not in ("bootstrap", "native"): + msger.error("Invalid runtime mode: %s" % runtime) - def _parse_bootstrap(self, bsconf): - if not bsconf or not os.path.exists(bsconf): - self.bootstrap['enable'] = False - return - - parser = ConfigParser.SafeConfigParser() - parser.read(bsconf) - - for section in parser.sections(): - if section == "main": - self.bootstrap.update(dict(parser.items(section))) - else: - self.bootstrap['distros'].append(section) - distro = section.lower() - self.bootstrap[distro] = dict(parser.items(section)) - for item in ('optional', 'packages'): - if not item in self.bootstrap[distro]: - continue - pks = self.bootstrap[distro][item].replace('\n', ' ') - self.bootstrap[distro][item] = pks.split() - - # update bootstrap options - if self.bootstrap['enable'] not in (True, False): - try: - self.bootstrap['enable'] = parser.getboolean('main', 'enable') - except: - self.bootstrap['enable'] = False - if not self.bootstrap['distro_name']: - self.bootstrap['distro_name'] = self.common['distro_name'] - if self.bootstrap['distro_name'] not in self.bootstrap['distros']: - self.bootstrap['enable'] = False + if misc.get_distro()[0] in ("tizen", "Tizen"): + runtime = "native" + self.create['runtime'] = runtime configmgr = ConfigMgr() diff --git a/mic/creator.py b/mic/creator.py index d6a1768..9e57841 100644 --- a/mic/creator.py +++ b/mic/creator.py @@ -92,8 +92,7 @@ class Creator(cmdln.Cmdln): help='Path for local pkgs(rpms) to be installed') optparser.add_option('', '--runtime', type='string', dest='runtime', default=None, - #help='Specify runtime mode, avaiable: bootstrap') - help=SUPPRESS_HELP) + help='Specify runtime mode, avaiable: bootstrap, native') # --taring-to is alias to --pack-to optparser.add_option('', '--taring-to', type='string', dest='pack_to', default=None, @@ -207,7 +206,7 @@ class Creator(cmdln.Cmdln): configmgr.create['pkgmgr'] = self.options.pkgmgr if self.options.runtime: - configmgr.create['runtime'] = self.options.runtime + configmgr.set_runtime(self.options.runtime) if self.options.pack_to is not None: configmgr.create['pack_to'] = self.options.pack_to diff --git a/mic/rt_util.py b/mic/rt_util.py index 918c6d5..3849c3c 100644 --- a/mic/rt_util.py +++ b/mic/rt_util.py @@ -47,32 +47,34 @@ def bootstrap_mic(argv=None): cropts = configmgr.create bsopts = configmgr.bootstrap distro = bsopts['distro_name'].lower() - if distro not in bsopts['distros'] or \ - 'packages' not in bsopts[distro]: - msger.info("Use native running for distro don't support bootstrap") - return rootdir = bsopts['rootdir'] - pkglist = bsopts[distro]['packages'] + pkglist = bsopts['packages'] cwd = os.getcwd() # create bootstrap and run mic in bootstrap bsenv = bootstrap.Bootstrap(rootdir, distro, cropts['arch']) bsenv.logfile = cropts['logfile'] - if 'optional' in bsopts[distro]: - optlist = bsopts[distro]['optional'] + if 'optional' in bsopts: + optlist = bsopts['optional'] else: optlist = [] try: msger.info("Creating %s bootstrap ..." % distro) bsenv.create(cropts['repomd'], pkglist, optlist) + + # bootstrap is relocated under "bootstrap" + if os.path.exists(os.path.join(rootdir, "bootstrap")): + rootdir = os.path.join(rootdir, "bootstrap") + + bsenv.dirsetup(rootdir) sync_mic(rootdir) msger.info("Start mic in bootstrap: %s\n" % rootdir) bindmounts = get_bindmounts(cropts) - ret = bsenv.run(argv, cwd, bindmounts) + ret = bsenv.run(argv, cwd, rootdir, bindmounts) except errors.BootstrapError, err: msger.warning('\n%s' % err) @@ -170,18 +172,6 @@ def sync_mic(bootstrap, binpth = '/usr/bin/mic', except (OSError, IOError), err: raise errors.BootstrapError(err) - # clean stuff: - # bootstrap.conf, disable bootstrap mode inside bootstrap - clrpaths = [ - '/etc/mic/bootstrap.conf', - ] - - for pth in clrpaths: - try: - os.unlink(_path(pth)) - except: - pass - # auto select backend conf_str = file(_path(conf)).read() conf_str = re.sub("pkgmgr\s*=\s*.*", "pkgmgr=auto", conf_str) diff --git a/plugins/imager/fs_plugin.py b/plugins/imager/fs_plugin.py index e2ee82b..cd597ab 100644 --- a/plugins/imager/fs_plugin.py +++ b/plugins/imager/fs_plugin.py @@ -49,7 +49,7 @@ class FsPlugin(ImagerPlugin): creatoropts = configmgr.create ksconf = args[0] - if configmgr.bootstrap['enable']: + if creatoropts['runtime'] == 'bootstrap': configmgr._ksconf = ksconf rt_util.bootstrap_mic() diff --git a/plugins/imager/livecd_plugin.py b/plugins/imager/livecd_plugin.py index 275f5c1..2d79aa4 100644 --- a/plugins/imager/livecd_plugin.py +++ b/plugins/imager/livecd_plugin.py @@ -45,7 +45,7 @@ class LiveCDPlugin(ImagerPlugin): creatoropts = configmgr.create ksconf = args[0] - if configmgr.bootstrap['enable']: + if creatoropts['runtime'] == 'bootstrap': configmgr._ksconf = ksconf rt_util.bootstrap_mic() diff --git a/plugins/imager/liveusb_plugin.py b/plugins/imager/liveusb_plugin.py index 02cfb45..08e293a 100644 --- a/plugins/imager/liveusb_plugin.py +++ b/plugins/imager/liveusb_plugin.py @@ -47,7 +47,7 @@ class LiveUSBPlugin(ImagerPlugin): creatoropts = configmgr.create ksconf = args[0] - if configmgr.bootstrap['enable']: + if creatoropts['runtime'] == "bootstrap": configmgr._ksconf = ksconf rt_util.bootstrap_mic() diff --git a/plugins/imager/loop_plugin.py b/plugins/imager/loop_plugin.py index c1bccba..505ccf8 100644 --- a/plugins/imager/loop_plugin.py +++ b/plugins/imager/loop_plugin.py @@ -54,7 +54,7 @@ class LoopPlugin(ImagerPlugin): creatoropts = configmgr.create ksconf = args[0] - if configmgr.bootstrap['enable']: + if creatoropts['runtime'] == "bootstrap": configmgr._ksconf = ksconf rt_util.bootstrap_mic() diff --git a/plugins/imager/raw_plugin.py b/plugins/imager/raw_plugin.py index 3cb15ed..4c432a6 100644 --- a/plugins/imager/raw_plugin.py +++ b/plugins/imager/raw_plugin.py @@ -54,7 +54,7 @@ class RawPlugin(ImagerPlugin): creatoropts = configmgr.create ksconf = args[0] - if configmgr.bootstrap['enable']: + if creatoropts['runtime'] == "bootstrap": configmgr._ksconf = ksconf rt_util.bootstrap_mic() diff --git a/setup.py b/setup.py index 83ad835..f68c1c0 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,6 @@ else: etc_prefix = os.path.join(prefix, 'etc') conffile = 'distfiles/mic.conf' -bsconf = 'distfiles/bootstrap.conf' if os.path.isfile('%s/mic/mic.conf' % etc_prefix): conffile += '.new' @@ -102,7 +101,7 @@ try: packages = PACKAGES, data_files = [("%s/lib/mic/plugins/imager" % prefix, IMAGER_PLUGINS), ("%s/lib/mic/plugins/backend" % prefix, BACKEND_PLUGINS), - ("%s/mic" % etc_prefix, [conffile, bsconf])] + ("%s/mic" % etc_prefix, [conffile])] ) finally: # remove dynamic file distfiles/mic.conf -- 2.7.4