From: Gui Chen Date: Tue, 15 Jan 2013 08:05:06 +0000 (+0800) Subject: split out the download stuff as one python module X-Git-Tag: 0.16~44 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=be2d409b9dc2f8d7a29eeeefbb0e06b8224d1d2c;p=platform%2Fupstream%2Fmic.git split out the download stuff as one python module mix the download code into rpmmisc is mess, try to make it stand alone, now the download stuff like myurlgrabb, TextProcess can be treated in one owned module Signed-off-by: Gui Chen --- diff --git a/mic/imager/baseimager.py b/mic/imager/baseimager.py index 293c208..ed5412a 100644 --- a/mic/imager/baseimager.py +++ b/mic/imager/baseimager.py @@ -32,7 +32,7 @@ import rpm from mic import kickstart from mic import msger from mic.utils.errors import CreatorError, Abort -from mic.utils import misc, rpmmisc, runner, fs_related as fs +from mic.utils import misc, grabber, runner, fs_related as fs class BaseImageCreator(object): """Installs a system to a chroot directory. @@ -861,7 +861,7 @@ class BaseImageCreator(object): if not os.path.exists(fpath): # download pkgs try: - fpath = rpmmisc.myurlgrab(url, fpath, proxies, None) + fpath = grabber.myurlgrab(url, fpath, proxies, None) except CreatorError: raise diff --git a/mic/utils/grabber.py b/mic/utils/grabber.py new file mode 100644 index 0000000..31e9d04 --- /dev/null +++ b/mic/utils/grabber.py @@ -0,0 +1,90 @@ +#!/usr/bin/python + +import os +import sys +import rpm +import fcntl +import struct +import termios + +from mic import msger +from mic.utils import runner +from mic.utils.errors import CreatorError + +from urlgrabber import grabber +from urlgrabber import __version__ as grabber_version + +if rpm.labelCompare(grabber_version.split('.'), '3.9.0'.split('.')) == -1: + msger.warning("Version of python-urlgrabber is %s, lower than '3.9.0', " + "you may encounter some network issues" % grabber_version) + +def myurlgrab(url, filename, proxies, progress_obj = None): + g = grabber.URLGrabber() + if progress_obj is None: + progress_obj = TextProgress() + + if url.startswith("file:/"): + filepath = "/%s" % url.replace("file:", "").lstrip('/') + if not os.path.exists(filepath): + raise CreatorError("URLGrabber error: can't find file %s" % url) + if url.endswith('.rpm'): + return filepath + else: + # untouch repometadata in source path + runner.show(['cp', '-f', filepath, filename]) + + else: + try: + filename = g.urlgrab(url = url, filename = filename, + ssl_verify_host = False, ssl_verify_peer = False, + proxies = proxies, http_headers = (('Pragma', 'no-cache'),), + quote = 0, progress_obj = progress_obj) + except grabber.URLGrabError, e: + raise CreatorError("URLGrabber error: %s" % url) + + return filename + +def terminal_width(fd=1): + """ Get the real terminal width """ + try: + buf = 'abcdefgh' + buf = fcntl.ioctl(fd, termios.TIOCGWINSZ, buf) + return struct.unpack('hhhh', buf)[1] + except: # IOError + return 80 + +def truncate_url(url, width): + return os.path.basename(url)[0:width] + +class TextProgress(object): + # make the class as singleton + _instance = None + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = super(TextProgress, cls).__new__(cls, *args, **kwargs) + + return cls._instance + + def __init__(self, totalnum = None): + self.total = totalnum + self.counter = 1 + + def start(self, filename, url, *args, **kwargs): + self.url = url + self.termwidth = terminal_width() + msger.info("\r%-*s" % (self.termwidth, " ")) + if self.total is None: + msger.info("\rRetrieving %s ..." % truncate_url(self.url, self.termwidth - 15)) + else: + msger.info("\rRetrieving %s [%d/%d] ..." % (truncate_url(self.url, self.termwidth - 25), self.counter, self.total)) + + def update(self, *args): + pass + + def end(self, *args): + if self.counter == self.total: + msger.raw("\n") + + if self.total is not None: + self.counter += 1 + diff --git a/mic/utils/misc.py b/mic/utils/misc.py index f99e39c..31bf00e 100644 --- a/mic/utils/misc.py +++ b/mic/utils/misc.py @@ -42,7 +42,7 @@ xmlparse = cElementTree.parse from errors import * from fs_related import * -from rpmmisc import myurlgrab +from grabber import myurlgrab from proxy import get_proxy_for import runner diff --git a/mic/utils/rpmmisc.py b/mic/utils/rpmmisc.py index 5cd7b46..37a22eb 100644 --- a/mic/utils/rpmmisc.py +++ b/mic/utils/rpmmisc.py @@ -15,89 +15,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import os, sys, re -import fcntl -import struct -import termios +import os +import sys +import re import rpm from mic import msger from .errors import CreatorError from .proxy import get_proxy_for import runner -from urlgrabber import grabber, __version__ as grabber_version -if rpm.labelCompare(grabber_version.split('.'), '3.9.0'.split('.')) == -1: - msger.warning("Version of python-urlgrabber is %s, lower than '3.9.0', " - "you may encounter some network issues" % grabber_version) - -def myurlgrab(url, filename, proxies, progress_obj = None): - g = grabber.URLGrabber() - if progress_obj is None: - progress_obj = TextProgress() - - if url.startswith("file:/"): - filepath = "/%s" % url.replace("file:", "").lstrip('/') - if not os.path.exists(filepath): - raise CreatorError("URLGrabber error: can't find file %s" % url) - if url.endswith('.rpm'): - return filepath - else: - # untouch repometadata in source path - runner.show(['cp', '-f', filepath, filename]) - - else: - try: - filename = g.urlgrab(url = url, filename = filename, - ssl_verify_host = False, ssl_verify_peer = False, - proxies = proxies, http_headers = (('Pragma', 'no-cache'),), - quote = 0, progress_obj = progress_obj) - except grabber.URLGrabError, e: - raise CreatorError("URLGrabber error: %s" % url) - - return filename - -def terminal_width(fd=1): - """ Get the real terminal width """ - try: - buf = 'abcdefgh' - buf = fcntl.ioctl(fd, termios.TIOCGWINSZ, buf) - return struct.unpack('hhhh', buf)[1] - except: # IOError - return 80 - -def truncate_url(url, width): - return os.path.basename(url)[0:width] - -class TextProgress(object): - # make the class as singleton - _instance = None - def __new__(cls, *args, **kwargs): - if not cls._instance: - cls._instance = super(TextProgress, cls).__new__(cls, *args, **kwargs) - - return cls._instance - - def __init__(self, totalnum = None): - self.total = totalnum - self.counter = 1 - - def start(self, filename, url, *args, **kwargs): - self.url = url - self.termwidth = terminal_width() - msger.info("\r%-*s" % (self.termwidth, " ")) - if self.total is None: - msger.info("\rRetrieving %s ..." % truncate_url(self.url, self.termwidth - 15)) - else: - msger.info("\rRetrieving %s [%d/%d] ..." % (truncate_url(self.url, self.termwidth - 25), self.counter, self.total)) - - def update(self, *args): - pass - - def end(self, *args): - if self.counter == self.total: - msger.raw("\n") - - if self.total is not None: - self.counter += 1 class RPMInstallCallback: """ Command line callback class for callbacks from the RPM library. diff --git a/plugins/backend/yumpkgmgr.py b/plugins/backend/yumpkgmgr.py index 33e354e..910f1cf 100644 --- a/plugins/backend/yumpkgmgr.py +++ b/plugins/backend/yumpkgmgr.py @@ -28,6 +28,7 @@ import yum from mic import msger from mic.kickstart import ksparser from mic.utils import misc, rpmmisc +from mic.utils.grabber import TextProgress from mic.utils.proxy import get_proxy_for from mic.utils.errors import CreatorError from mic.imager.baseimager import BaseImageCreator @@ -404,8 +405,7 @@ class Yum(BackendPlugin, yum.YumBase): try: repos = self.repos.listEnabled() for repo in repos: - repo.setCallback( - rpmmisc.TextProgress(total_count - cached_count)) + repo.setCallback(TextProgress(total_count - cached_count)) self.downloadPkgs(dlpkgs) # FIXME: sigcheck? diff --git a/plugins/backend/zypppkgmgr.py b/plugins/backend/zypppkgmgr.py index 6f7807c..468a8b7 100755 --- a/plugins/backend/zypppkgmgr.py +++ b/plugins/backend/zypppkgmgr.py @@ -30,6 +30,7 @@ if not hasattr(zypp, 'PoolQuery') or \ from mic import msger from mic.kickstart import ksparser from mic.utils import misc, rpmmisc, runner, fs_related +from mic.utils.grabber import myurlgrab, TextProgress from mic.utils.proxy import get_proxy_for from mic.utils.errors import CreatorError, RepoError, RpmError from mic.imager.baseimager import BaseImageCreator @@ -628,7 +629,7 @@ class Zypp(BackendPlugin): def downloadPkgs(self, package_objects, count): localpkgs = self.localpkgs.keys() - progress_obj = rpmmisc.TextProgress(count) + progress_obj = TextProgress(count) for po in package_objects: if po.name() in localpkgs: @@ -647,10 +648,7 @@ class Zypp(BackendPlugin): proxies = self.get_proxies(po) try: - filename = rpmmisc.myurlgrab(url, - filename, - proxies, - progress_obj) + filename = myurlgrab(url, filename, proxies, progress_obj) except CreatorError: self.close() raise