2 # Copyright (c) 2013 Intel Inc.
4 # This program is free software; you can redistribute it and/or modify it
5 # under the terms of the GNU General Public License as published by the Free
6 # Software Foundation; version 2 of the License
8 # This program is distributed in the hope that it will be useful, but
9 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
10 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc., 59
15 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 # Following messages should be disabled in pylint
18 # * Used * or ** magic (W0142)
19 # * Unused variable (W0612)
20 # * Used builtin function (W0141)
21 # * Invalid name for type (C0103)
22 # * Popen has no '%s' member (E1101)
23 # pylint: disable=W0142, W0612, W0141, C0103, E1101
25 """ Compression and Archiving
27 Utility functions for creating archive files (tarballs, zip files, etc)
28 and compressing files (gzip, bzip2, lzop, etc)
38 "get_compress_formats",
41 "get_archive_formats",
42 "get_archive_suffixes",
49 # TODO: refine Error class for archive/extract
51 def which(binary, path=None):
52 """ Find 'binary' in the directories listed in 'path'
54 @binary: the executable file to find
55 @path: the suposed path to search for, use $PATH if None
56 @retval: the absolute path if found, otherwise None
59 path = os.environ["PATH"]
60 for apath in path.split(os.pathsep):
61 fpath = os.path.join(apath, binary)
62 if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
66 def _call_external(cmdln_or_args):
67 """ Wapper for subprocess calls.
69 @cmdln_or_args: command line to be joined before execution.
70 @retval: a tuple (returncode, outdata, errdata).
72 if isinstance(cmdln_or_args, list):
76 msger.info("Running command: " + " ".join(cmdln_or_args))
78 proc = subprocess.Popen(cmdln_or_args, shell=shell,
79 stdout=subprocess.PIPE,
80 stderr=subprocess.STDOUT)
81 (outdata, errdata) = proc.communicate()
83 return (proc.returncode, outdata, errdata)
85 def _do_gzip(input_name, compression=True):
86 """ Compress/decompress the file with 'gzip' utility.
88 @input_name: the file name to compress/decompress
89 @compress: True for compressing, False for decompressing
90 @retval: the path of the compressed/decompressed file
92 if which("pigz") is not None:
98 cmdln = [compressor, "-f", input_name]
100 cmdln = [compressor, "-d", "-f", input_name]
102 _call_external(cmdln)
105 output_name = input_name + ".gz"
107 # suppose that file name is suffixed with ".gz"
108 output_name = os.path.splitext(input_name)[0]
112 def _do_bzip2(input_name, compression=True):
113 """ Compress/decompress the file with 'bzip2' utility.
115 @input_name: the file name to compress/decompress
116 @compress: True for compressing, False for decompressing
117 @retval: the path of the compressed/decompressed file
119 if which("pbzip2") is not None:
120 compressor = "pbzip2"
125 cmdln = [compressor, "-f", input_name]
127 cmdln = [compressor, "-d", "-f", input_name]
129 _call_external(cmdln)
132 output_name = input_name + ".bz2"
134 # suppose that file name is suffixed with ".bz2"
135 output_name = os.path.splitext(input_name)[0]
139 def _do_lzop(input_name, compression=True):
140 """ Compress/decompress the file with 'lzop' utility.
142 @input_name: the file name to compress/decompress
143 @compress: True for compressing, False for decompressing
144 @retval: the path of the compressed/decompressed file
149 cmdln = [compressor, "-f", "-U", input_name]
151 cmdln = [compressor, "-d", "-f", "-U", input_name]
153 _call_external(cmdln)
156 output_name = input_name + ".lzo"
158 # suppose that file name is suffixed with ".lzo"
159 output_name = os.path.splitext(input_name)[0]
163 _COMPRESS_SUFFIXES = {
166 ".bz2" : [".bz2", ".bz"],
167 ".tar.lzo" : [".tar.lzo", ".tzo"],
168 ".tar.gz" : [".tar.gz", ".tgz", ".taz"],
169 ".tar.bz2" : [".tar.bz2", ".tbz", ".tbz2", ".tar.bz"],
172 _COMPRESS_FORMATS = {
178 def get_compress_formats():
179 """ Get the list of the supported compression formats
181 @retval: a list contained supported compress formats
183 return _COMPRESS_FORMATS.keys()
185 def get_compress_suffixes():
186 """ Get the list of the support suffixes
188 @retval: a list contained all suffixes
191 for key in _COMPRESS_SUFFIXES.keys():
192 suffix = _COMPRESS_SUFFIXES[key]
194 suffixes.extend(suffix)
198 def compress(file_path, compress_format):
199 """ Compress a given file
201 @file_path: the path of the file to compress
202 @compress_format: the compression format
203 @retval: the path of the compressed file
205 if not os.path.isfile(file_path):
206 raise OSError, "can't compress a file not existed: '%s'" % file_path
209 func = _COMPRESS_FORMATS[compress_format]
211 raise ValueError, "unknown compress format '%s'" % compress_format
212 return func(file_path, True)
214 def decompress(file_path, decompress_format=None):
215 """ Decompess a give file
217 @file_path: the path of the file to decompress
218 @decompress_format: the format for decompression, None for auto detection
219 @retval: the path of the decompressed file
221 if not os.path.isfile(file_path):
222 raise OSError, "can't decompress a file not existed: '%s'" % file_path
224 (file_name, file_ext) = os.path.splitext(file_path)
225 for key, suffixes in _COMPRESS_SUFFIXES.iteritems():
226 if file_ext in suffixes:
230 if file_path != (file_name + file_ext):
231 shutil.move(file_path, file_name + file_ext)
232 file_path = file_name + file_ext
234 if not decompress_format:
235 decompress_format = os.path.splitext(file_path)[1].lstrip(".")
238 func = _COMPRESS_FORMATS[decompress_format]
240 raise ValueError, "unknown decompress format '%s'" % decompress_format
241 return func(file_path, False)
244 def _do_tar(archive_name, target_name):
245 """ Archive the directory or the file with 'tar' utility
247 @archive_name: the name of the tarball file
248 @target_name: the name of the target to tar
249 @retval: the path of the archived file
252 if os.path.isdir(target_name):
253 target_dir = target_name
255 for target_file in os.listdir(target_dir):
256 file_list.append(target_file)
258 target_dir = os.path.dirname(target_name)
259 target_name = os.path.basename(target_name)
260 file_list.append(target_name)
261 cmdln = ["tar", "-C", target_dir, "-cf", archive_name]
262 cmdln.extend(file_list)
263 _call_external(cmdln)
267 def _do_untar(archive_name, target_dir=None):
268 """ Unarchive the archived file with 'tar' utility
270 @archive_name: the name of the archived file
271 @target_dir: the directory which the target locates
272 raise exception if fail to extract archive
275 target_dir = os.getcwd()
276 cmdln = ["tar", "-C", target_dir, "-xf", archive_name]
277 (returncode, stdout, stderr) = _call_external(cmdln)
279 raise OSError, os.linesep.join([stdout, stderr])
281 def _imp_tarfile(archive_name, target_name):
282 """ Archive the directory or the file with tarfile module
284 @archive_name: the name of the tarball file
285 @target_name: the name of the target to tar
286 @retval: the path of the archived file
290 msger.info("Taring files to %s using tarfile module" % archive_name)
291 tar = tarfile.open(archive_name, 'w')
292 if os.path.isdir(target_name):
293 for child in os.listdir(target_name):
294 tar.add(os.path.join(target_name, child), child)
296 tar.add(target_name, os.path.basename(target_name))
301 def _make_tarball(archive_name, target_name, compressor=None):
302 """ Create a tarball from all the files under 'target_name' or itself.
304 @archive_name: the name of the archived file to create
305 @target_name: the directory or the file name to archive
306 @compressor: callback function to compress the tarball
307 @retval: indicate the compressing result
309 archive_dir = os.path.dirname(archive_name)
310 tarball_name = tempfile.mktemp(suffix=".tar", dir=archive_dir)
312 if which("tar") is not None:
313 _do_tar(tarball_name, target_name)
315 _imp_tarfile(tarball_name, target_name)
318 tarball_name = compressor(tarball_name, True)
320 shutil.move(tarball_name, archive_name)
322 return os.path.exists(archive_name)
324 def _extract_tarball(archive_name, target_dir, compressor=None):
325 """ Extract a tarball to a target directory
327 @archive_name: the name of the archived file to extract
328 @target_dir: the directory of the extracted target
331 _do_untar(archive_name, target_dir)
333 def _make_zipfile(archive_name, target_name):
334 """ Create a zip file from all the files under 'target_name' or itself.
336 @archive_name: the name of the archived file
337 @target_name: the directory or the file name to archive
338 @retval: indicate the archiving result
342 msger.info("Zipping files to %s using zipfile module" % archive_name)
343 arv = zipfile.ZipFile(archive_name, 'w', compression=zipfile.ZIP_DEFLATED)
345 if os.path.isdir(target_name):
346 for dirpath, dirname, filenames in os.walk(target_name):
347 for filename in filenames:
348 filepath = os.path.normpath(os.path.join(dirpath, filename))
349 arcname = os.path.relpath(filepath, target_name)
350 if os.path.isfile(filepath):
351 arv.write(filepath, arcname)
353 arv.write(target_name, os.path.basename(target_name))
357 return os.path.exists(archive_name)
359 _ARCHIVE_SUFFIXES = {
362 "lzotar": [".tzo", ".tar.lzo"],
363 "gztar" : [".tgz", ".taz", ".tar.gz"],
364 "bztar" : [".tbz", ".tbz2", ".tar.bz", ".tar.bz2"],
368 "zip" : ( _make_zipfile, {} ),
369 "tar" : ( _make_tarball, {"compressor" : None} ),
370 "lzotar": ( _make_tarball, {"compressor" : _do_lzop} ),
371 "gztar" : ( _make_tarball, {"compressor" : _do_gzip} ),
372 "bztar" : ( _make_tarball, {"compressor" : _do_bzip2} ),
375 def get_archive_formats():
376 """ Get the list of the supported formats for archiving
378 @retval: a list contained archive formats
380 return _ARCHIVE_FORMATS.keys()
382 def get_archive_suffixes():
383 """ Get the list of the support suffixes
385 @retval: a list contained all suffixes
388 for name in _ARCHIVE_FORMATS.keys():
389 suffix = _ARCHIVE_SUFFIXES.get(name, None)
391 suffixes.extend(suffix)
395 def make_archive(archive_name, target_name):
396 """ Create an archive file (eg. tar or zip).
398 @archive_name: the name of the archived file
399 @target_name: the directory or the file to archive
400 @retval: the archiving result
402 if not os.path.exists(target_name):
403 raise OSError, "archive object does not exist: '%s'" % target_name
405 for aformat, suffixes in _ARCHIVE_SUFFIXES.iteritems():
406 if filter(archive_name.endswith, suffixes):
407 archive_format = aformat
410 raise ValueError, "unknown archive suffix '%s'" % archive_name
413 func, kwargs = _ARCHIVE_FORMATS[archive_format]
415 raise ValueError, "unknown archive format '%s'" % archive_format
417 archive_name = os.path.abspath(archive_name)
418 target_name = os.path.abspath(target_name)
420 archive_dir = os.path.dirname(archive_name)
421 if not os.path.exists(archive_dir):
422 os.makedirs(archive_dir)
424 return func(archive_name, target_name, **kwargs)
426 def extract_archive(archive_name, target_name):
427 """ Extract the given file
429 @archive_name: the name of the archived file to extract
430 @target_name: the directory name where the target locates
431 raise exception if fail to extract archive
433 if not os.path.exists(archive_name):
434 raise OSError, "archived object does not exist: '%s'" % archive_name
436 archive_name = os.path.abspath(archive_name)
437 target_name = os.path.abspath(target_name)
439 if os.path.exists(target_name) and not os.path.isdir(target_name):
440 raise OSError, "%s should be directory where extracted files locate"\
443 if not os.path.exists(target_name):
444 os.makedirs(target_name)
446 _extract_tarball(archive_name, target_name)
448 packing = make_archive
449 compressing = compress