From 839c1654c954d98de36f5891ff6c64a6fc422e6a Mon Sep 17 00:00:00 2001 From: Artem Bityutskiy Date: Wed, 29 Jan 2014 19:05:06 +0200 Subject: [PATCH] test_api_base: use external programs for compression Stop using internal python modules for compressing test files, and just like we did in the previous commit, use external programs for this. Change-Id: I75dd6461d598d1d4b0b4096b65d74f08c44adc75 Signed-off-by: Artem Bityutskiy --- tests/test_api_base.py | 108 +++++++++++++---------------------------- 1 file changed, 35 insertions(+), 73 deletions(-) diff --git a/tests/test_api_base.py b/tests/test_api_base.py index 23ca625..934ee5f 100644 --- a/tests/test_api_base.py +++ b/tests/test_api_base.py @@ -30,6 +30,9 @@ import os import tempfile import filecmp import itertools +import subprocess +from tests import helpers +from bmaptools import BmapHelpers, BmapCreate, Filemap # This is a work-around for Centos 6 try: @@ -37,9 +40,6 @@ try: except ImportError: import unittest -from tests import helpers -from bmaptools import BmapCreate, Filemap, TransRead - class Error(Exception): """A class for exceptions generated by this test.""" pass @@ -62,6 +62,7 @@ def _compare_holes(file1, file2): raise Error("mismatch for hole %d-%d, it is %d-%d in file2" % (range1[0], range1[1], range2[0], range2[1])) + def _generate_compressed_files(file_path, delete=True): """ This is a generator which yields compressed versions of a file @@ -71,86 +72,47 @@ def _generate_compressed_files(file_path, delete=True): generator yields have to be automatically deleted. """ - import bz2 - import gzip - import tarfile - import shutil - - lzma_present = True - try: - import lzma - except ImportError: - try: - from backports import lzma # pylint: disable=F0401 - except ImportError: - lzma_present = False - # Make sure the temporary files start with the same name as 'file_obj' in # order to simplify debugging. prefix = os.path.splitext(os.path.basename(file_path))[0] + '.' # Put the temporary files in the directory with 'file_obj' directory = os.path.dirname(file_path) - # Generate an uncompressed version of the file - tmp_file_obj = tempfile.NamedTemporaryFile('wb+', prefix=prefix, - delete=delete, dir=directory, - suffix='.uncompressed') - shutil.copyfileobj(TransRead.TransRead(file_path), tmp_file_obj) - tmp_file_obj.flush() - yield tmp_file_obj.name - tmp_file_obj.close() - - # Generate a .bz2 version of the file - tmp_file_obj = tempfile.NamedTemporaryFile('wb+', prefix=prefix, - delete=delete, dir=directory, - suffix='.bz2') - bz2_file_obj = bz2.BZ2File(tmp_file_obj.name, 'wb') - shutil.copyfileobj(TransRead.TransRead(file_path), bz2_file_obj) - bz2_file_obj.close() - yield bz2_file_obj.name - tmp_file_obj.close() - - # Generate a .gz version of the file - tmp_file_obj = tempfile.NamedTemporaryFile('wb+', prefix=prefix, - delete=delete, dir=directory, - suffix='.gz') - gzip_file_obj = gzip.GzipFile(tmp_file_obj.name, 'wb') - shutil.copyfileobj(TransRead.TransRead(file_path), gzip_file_obj) - gzip_file_obj.close() - yield gzip_file_obj.name - tmp_file_obj.close() - - if lzma_present: - # Generate an .xz version of the file + compressors = [("bzip2", None, ".bz2", "-c -k"), + ("pbzip2", None, ".p.bz2", "-c -k"), + ("gzip", None, ".gz", "-c"), + ("pigz", None, ".p.gz", "-c -k"), + ("xz", None, ".xz", "-c -k"), + ("lzop", None, ".lzo", "-c -k"), + # The "-P -C /" trick is used to avoid silly warnings: + # "tar: Removing leading `/' from member names" + ("bzip2", "tar", ".tar.bz2", "-c -j -O -P -C /"), + ("gzip", "tar", ".tar.gz", "-c -z -O -P -C /"), + ("xz", "tar", ".tar.xz", "-c -J -O -P -C /"), + ("lzop", "tar", ".tar.lzo", "-c --lzo -O -P -C /")] + + for decompressor, archiver, suffix, options in compressors: + if not BmapHelpers.program_is_available(decompressor): + continue + if archiver and not BmapHelpers.program_is_available(archiver): + continue + tmp_file_obj = tempfile.NamedTemporaryFile('wb+', prefix=prefix, delete=delete, dir=directory, - suffix='.xz') - lzma_file_obj = lzma.LZMAFile(tmp_file_obj.name, 'wb') - shutil.copyfileobj(TransRead.TransRead(file_path), lzma_file_obj) - lzma_file_obj.close() - yield lzma_file_obj.name + suffix=suffix) + + if archiver: + args = archiver + " " + options + " " + file_path + else: + args = decompressor + " " + options + " " + file_path + child_process = subprocess.Popen(args, shell=True, + stderr=subprocess.PIPE, + stdout=tmp_file_obj) + child_process.wait() + tmp_file_obj.flush() + yield tmp_file_obj.name tmp_file_obj.close() - # Generate a tar.gz version of the file - tmp_file_obj = tempfile.NamedTemporaryFile('wb+', prefix=prefix, - delete=delete, dir=directory, - suffix='.tar.gz') - tgz_file_obj = tarfile.open(tmp_file_obj.name, "w:gz") - tgz_file_obj.add(file_path) - tgz_file_obj.close() - yield tgz_file_obj.name - tmp_file_obj.close() - - # Generate a tar.bz2 version of the file - tmp_file_obj = tempfile.NamedTemporaryFile('wb+', prefix=prefix, - delete=delete, dir=directory, - suffix='.tar.bz2') - tbz2_file_obj = tarfile.open(tmp_file_obj.name, "w:bz2") - tbz2_file_obj.add(file_path) - tbz2_file_obj.close() - yield tbz2_file_obj.name - tmp_file_obj.close() - def _do_test(image, image_size, delete=True): """ A basic test for the bmap creation and copying functionality. It first -- 2.34.1