Misc: Add cr_decompress_file_with_stat function (in Python: createrepo_c.decompress_f...
authorTomas Mlcoch <tmlcoch@redhat.com>
Thu, 10 Oct 2013 12:58:28 +0000 (14:58 +0200)
committerTomas Mlcoch <tmlcoch@redhat.com>
Thu, 10 Oct 2013 12:58:28 +0000 (14:58 +0200)
14 files changed:
src/compression_wrapper.c
src/misc.c
src/misc.h
src/python/__init__.py
src/python/createrepo_cmodule.c
src/python/misc-py.c
src/python/misc-py.h
tests/fixtures.h
tests/python/tests/fixtures.py
tests/python/tests/test_misc.py
tests/test_compression_wrapper.c
tests/test_misc.c
tests/test_sqlite.c
tests/testdata/test_files/text_file.gz [new file with mode: 0644]

index e7f0e62..a773c9e 100644 (file)
@@ -894,6 +894,19 @@ cr_read(CR_FILE *cr_file, void *buffer, unsigned int len, GError **err)
     assert(!err || (ret == CR_CW_ERR && *err != NULL)
            || (ret != CR_CW_ERR && *err == NULL));
 
+    if (cr_file->stat && ret != CR_CW_ERR) {
+        GError *tmp_err = NULL;
+        cr_file->stat->size += ret;
+        if (cr_file->checksum_ctx) {
+            GError *tmp_err = NULL;
+            cr_checksum_update(cr_file->checksum_ctx, buffer, ret, &tmp_err);
+            if (tmp_err) {
+                g_propagate_error(err, tmp_err);
+                return CR_CW_ERR;
+            }
+        }
+    }
+
     return ret;
 }
 
index c7a6112..7695e34 100644 (file)
@@ -438,7 +438,7 @@ cr_compress_file_with_stat(const char *src,
 
     // Src must be a file NOT a directory
     if (!g_file_test(src, G_FILE_TEST_IS_REGULAR)) {
-        g_debug("%s: Source (%s) must be directory!", __func__, src);
+        g_debug("%s: Source (%s) must be a regular file!", __func__, src);
         g_set_error(err, CR_MISC_ERROR, CRE_NOFILE,
                     "Not a regular file: %s", src);
         return CRE_NOFILE;
@@ -510,6 +510,121 @@ compress_file_cleanup:
 }
 
 
+int
+cr_decompress_file_with_stat(const char *src,
+                             const char *in_dst,
+                             cr_CompressionType compression,
+                             cr_ContentStat *stat,
+                             GError **err)
+{
+    int ret = CRE_OK;
+    int readed;
+    char buf[BUFFER_SIZE];
+    FILE *new = NULL;
+    CR_FILE *orig = NULL;
+    gchar *dst = (gchar *) in_dst;
+    GError *tmp_err = NULL;
+
+    assert(src);
+    assert(!err || *err == NULL);
+
+    // Src must be a file NOT a directory
+    if (!g_file_test(src, G_FILE_TEST_IS_REGULAR)) {
+        g_debug("%s: Source (%s) must be a regular file!", __func__, src);
+        g_set_error(err, CR_MISC_ERROR, CRE_NOFILE,
+                    "Not a regular file: %s", src);
+        return CRE_NOFILE;
+    }
+
+    if (compression == CR_CW_AUTO_DETECT_COMPRESSION ||
+        compression == CR_CW_UNKNOWN_COMPRESSION)
+    {
+        compression = cr_detect_compression(src, NULL);
+    }
+
+    if (compression == CR_CW_UNKNOWN_COMPRESSION) {
+        g_set_error(err, CR_MISC_ERROR, CRE_UNKNOWNCOMPRESSION,
+                    "Cannot detect compression type");
+        return CRE_UNKNOWNCOMPRESSION;
+    }
+
+    const char *c_suffix = cr_compression_suffix(compression);
+
+    if (!in_dst || g_str_has_suffix(in_dst, "/")) {
+        char *filename = cr_get_filename(src);
+        if (g_str_has_suffix(filename, c_suffix)) {
+            filename = g_strndup(filename, strlen(filename) - strlen(c_suffix));
+        } else {
+            filename = g_strconcat(filename, ".decompressed", NULL);
+        }
+
+        if (!in_dst) {
+            // in_dst is NULL, use same dir as src
+            char *src_dir = g_strndup(src,
+                                strlen(src) - strlen(cr_get_filename(src)));
+            dst = g_strconcat(src_dir, filename, NULL);
+            g_free(src_dir);
+        } else {
+            // in_dst is dir
+            dst = g_strconcat(in_dst, filename, NULL);
+        }
+
+        g_free(filename);
+    }
+
+    orig = cr_sopen(src, CR_CW_MODE_READ, compression, stat, &tmp_err);
+    if (orig == NULL) {
+        g_debug("%s: Cannot open source file %s", __func__, src);
+        g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", src);
+        ret = CRE_IO;
+        goto compress_file_cleanup;
+    }
+
+    new = fopen(dst, "wb");
+    if (!new) {
+        g_debug("%s: Cannot open destination file %s (%s)",
+                __func__, dst, strerror(errno));
+        g_set_error(err, CR_MISC_ERROR, CRE_IO,
+                    "Cannot open %s: %s", src, strerror(errno));
+        ret = CRE_IO;
+        goto compress_file_cleanup;
+    }
+
+    while ((readed = cr_read(orig, buf, BUFFER_SIZE, &tmp_err)) > 0) {
+        if (tmp_err) {
+            g_debug("%s: Error while copy %s -> %s (%s)", __func__, src,
+                    dst, tmp_err->message);
+            g_propagate_prefixed_error(err, tmp_err,
+                                       "Error while read %s: ", src);
+            ret = CRE_IO;
+            goto compress_file_cleanup;
+        }
+
+        if (fwrite(buf, 1, readed, new) != readed) {
+            g_debug("%s: Error while copy %s -> %s (%s)",
+                    __func__, src, dst, strerror(errno));
+            g_set_error(err, CR_MISC_ERROR, CRE_IO,
+                        "Error while write %s: %s", dst, strerror(errno));
+            ret = CRE_IO;
+            goto compress_file_cleanup;
+        }
+    }
+
+compress_file_cleanup:
+
+    if (dst != in_dst)
+        g_free(dst);
+
+    if (orig)
+        cr_close(orig, NULL);
+
+    if (new)
+        fclose(new);
+
+    return ret;
+}
+
+
 
 int
 cr_download(CURL *handle,
index d105be2..ef36f4d 100644 (file)
@@ -176,6 +176,36 @@ int cr_compress_file_with_stat(const char *src,
                                cr_ContentStat *stat,
                                GError **err);
 
+/** Decompress file.
+ * @param SRC           source filename
+ * @param DST           destination (If dst is dir, filename of src without
+ *                      compression suffix (if present) is used.
+ *                      If dst is NULL, src without compression suffix is used)
+ *                      Otherwise ".decompressed" suffix is used
+ * @param COMTYPE       type of compression
+ * @param ERR           GError **
+ * @return              cr_Error return code
+ */
+#define cr_decompress_file(SRC, DST, COMTYPE, ERR) \
+                    cr_decompress_file_with_stat(SRC, DST, COMTYPE, NULL, ERR)
+
+/** Decompress file.
+ * @param src           source filename
+ * @param dst           destination (If dst is dir, filename of src without
+ *                      compression suffix (if present) is used.
+ *                      If dst is NULL, src without compression suffix is used)
+ *                      Otherwise ".decompressed" suffix is used
+ * @param comtype       type of compression
+ * @param stat          pointer to cr_ContentStat or NULL
+ * @param err           GError **
+ * @return              cr_Error return code
+ */
+int cr_decompress_file_with_stat(const char *src,
+                                 const char *dst,
+                                 cr_CompressionType comtype,
+                                 cr_ContentStat *stat,
+                                 GError **err);
+
 /** Better copy file. Source (src) could be remote address (http:// or ftp://).
  * @param src           source filename
  * @param dst           destination (if dst is dir, filename of src is used)
index 2751863..68dfe95 100644 (file)
@@ -250,6 +250,9 @@ checksum_type       = _createrepo_c.checksum_type
 def compress_file(src, dst, comtype, stat=None):
     return _createrepo_c.compress_file_with_stat(src, dst, comtype, stat)
 
+def decompress_file(src, dst, comtype, stat=None):
+    return _createrepo_c.decompress_file_with_stat(src, dst, comtype, stat)
+
 compression_suffix  = _createrepo_c.compression_suffix
 detect_compression  = _createrepo_c.detect_compression
 compression_type    = _createrepo_c.compression_type
index 4cb019c..ca28d8f 100644 (file)
@@ -64,6 +64,8 @@ static struct PyMethodDef createrepo_c_methods[] = {
         METH_VARARGS, checksum_type__doc__},
     {"compress_file_with_stat", (PyCFunction)py_compress_file_with_stat,
         METH_VARARGS, compress_file_with_stat__doc__},
+    {"decompress_file_with_stat",(PyCFunction)py_decompress_file_with_stat,
+        METH_VARARGS, decompress_file_with_stat__doc__},
     {"compression_suffix",      (PyCFunction)py_compression_suffix,
         METH_VARARGS, compression_suffix__doc__},
     {"detect_compression",      (PyCFunction)py_detect_compression,
index 91a2ad3..1a90ba3 100644 (file)
@@ -59,3 +59,35 @@ py_compress_file_with_stat(PyObject *self, PyObject *args)
 
     Py_RETURN_NONE;
 }
+
+PyObject *
+py_decompress_file_with_stat(PyObject *self, PyObject *args)
+{
+    int type;
+    char *src, *dst;
+    PyObject *py_contentstat = NULL;
+    cr_ContentStat *contentstat;
+    GError *tmp_err = NULL;
+
+    CR_UNUSED(self);
+
+    if (!PyArg_ParseTuple(args, "sziO:py_decompress_file", &src, &dst, &type,
+                          &py_contentstat))
+        return NULL;
+
+    if (!py_contentstat || py_contentstat == Py_None) {
+        contentstat = NULL;
+    } else {
+        contentstat = ContentStat_FromPyObject(py_contentstat);
+        if (!contentstat)
+            return NULL;
+    }
+
+    cr_decompress_file_with_stat(src, dst, type, contentstat, &tmp_err);
+    if (tmp_err) {
+        nice_exception(&tmp_err, NULL);
+        return NULL;
+    }
+
+    Py_RETURN_NONE;
+}
index 75f11c5..09625cf 100644 (file)
@@ -29,4 +29,12 @@ PyDoc_STRVAR(compress_file_with_stat__doc__,
 
 PyObject *py_compress_file_with_stat(PyObject *self, PyObject *args);
 
+PyDoc_STRVAR(decompress_file_with_stat__doc__,
+"decompress_file_with_stat(source, destination, compression_type, "
+"contentstat_object) -> None\n\n"
+"Decompress file. destination and contentstat_object could be None");
+
+PyObject *py_decompress_file_with_stat(PyObject *self, PyObject *args);
+
+
 #endif
index 02b46cd..fa94168 100644 (file)
 
 // Test files
 
-#define TEST_EMPTY_FILE         TEST_FILES_PATH"empty_file"
-#define TEST_TEXT_FILE          TEST_FILES_PATH"text_file"
-#define TEST_BINARY_FILE        TEST_FILES_PATH"binary_file"
+#define TEST_EMPTY_FILE          TEST_FILES_PATH"empty_file"
+#define TEST_TEXT_FILE           TEST_FILES_PATH"text_file"
+#define TEST_TEXT_FILE_SHA256SUM "2f395bdfa2750978965e4781ddf224c89646c7d7a1569b7ebb023b170f7bd8bb"
+#define TEST_TEXT_FILE_GZ        TEST_FILES_PATH"text_file.gz"
+#define TEST_BINARY_FILE         TEST_FILES_PATH"binary_file"
 
 // Other
 
index be29c74..46bb723 100644 (file)
@@ -86,6 +86,8 @@ FILE_BINARY = "binary_file"
 FILE_BINARY_PATH = os.path.join(TEST_FILES_PATH, FILE_BINARY)
 FILE_TEXT = "text_file"
 FILE_TEXT = os.path.join(TEST_FILES_PATH, FILE_TEXT)
+FILE_TEXT_SHA256SUM = "2f395bdfa2750978965e4781ddf224c89646c7d7a1569b7ebb023b170f7bd8bb"
+FILE_TEXT_GZ = FILE_TEXT+".gz"
 FILE_EMPTY = "empty_file"
 FILE_EMPTY = os.path.join(TEST_FILES_PATH, FILE_EMPTY)
 
index 9d578f0..7e8353a 100644 (file)
@@ -45,3 +45,37 @@ class TestCaseMisc(unittest.TestCase):
         self.assertEqual(set(os.listdir(self.tmpdir)),
                          set(['file.bz2', 'file.xz', 'file', 'foobar.gz']))
 
+    def test_decompress_file(self):
+        # Non exist file
+        self.assertRaises(IOError, cr.decompress_file,
+                          self.nofile, None, cr.BZ2)
+
+        tmpfile_gz_comp = os.path.join(self.tmpdir, "gzipedfile.gz")
+        shutil.copy(FILE_TEXT_GZ, tmpfile_gz_comp)
+        tmpfile_gz_comp_ns = os.path.join(self.tmpdir, "gzipedfile_no_suffix")
+        shutil.copy(FILE_TEXT_GZ, tmpfile_gz_comp_ns)
+
+        # Decompression - use the same name without suffix
+        dest = os.path.join(self.tmpdir, "gzipedfile")
+        cr.decompress_file(tmpfile_gz_comp, None, cr.GZ)
+        self.assertTrue(os.path.isfile(dest))
+
+        # Decompression - use the specific name
+        dest = os.path.join(self.tmpdir, "decompressed.file")
+        cr.decompress_file(tmpfile_gz_comp, dest, cr.GZ)
+        self.assertTrue(os.path.isfile(dest))
+
+        # Decompression - bad suffix by default
+        dest = os.path.join(self.tmpdir, "gzipedfile_no_suffix.decompressed")
+        cr.decompress_file(tmpfile_gz_comp_ns, None, cr.GZ)
+        self.assertTrue(os.path.isfile(dest))
+
+        # Decompression - with stat
+        stat = cr.ContentStat(cr.SHA256)
+        dest = os.path.join(self.tmpdir, "gzipedfile")
+        cr.decompress_file(tmpfile_gz_comp, None, cr.AUTO_DETECT_COMPRESSION, stat)
+        self.assertTrue(os.path.isfile(dest))
+        self.assertEqual(stat.checksum, FILE_TEXT_SHA256SUM)
+        self.assertEqual(stat.checksum_type, cr.SHA256)
+        self.assertEqual(stat.size, 910L)
+
index 0a3dfd9..badc4bf 100644 (file)
@@ -369,21 +369,18 @@ outputtest_cw_output(Outputtest *outputtest, gconstpointer test_data)
 
     // Plain
 
-    printf("Testing - plain\nwrite()\n");
     test_helper_cw_output(OUTPUT_TYPE_WRITE,  outputtest->tmp_filename,
                           CR_CW_NO_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
                           FILE_COMPRESSED_0_CONTENT_LEN);
     test_helper_cw_output(OUTPUT_TYPE_WRITE,  outputtest->tmp_filename,
                           CR_CW_NO_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
                           FILE_COMPRESSED_1_CONTENT_LEN);
-    printf("puts()\n");
     test_helper_cw_output(OUTPUT_TYPE_PUTS,   outputtest->tmp_filename,
                           CR_CW_NO_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
                           FILE_COMPRESSED_0_CONTENT_LEN);
     test_helper_cw_output(OUTPUT_TYPE_PUTS,   outputtest->tmp_filename,
                           CR_CW_NO_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
                           FILE_COMPRESSED_1_CONTENT_LEN);
-    printf("printf()\n");
     test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename,
                           CR_CW_NO_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
                           FILE_COMPRESSED_0_CONTENT_LEN);
@@ -393,21 +390,18 @@ outputtest_cw_output(Outputtest *outputtest, gconstpointer test_data)
 
     // Gz
 
-    printf("Testing - gz\nwrite()\n");
     test_helper_cw_output(OUTPUT_TYPE_WRITE,  outputtest->tmp_filename,
                           CR_CW_GZ_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
                           FILE_COMPRESSED_0_CONTENT_LEN);
     test_helper_cw_output(OUTPUT_TYPE_WRITE,  outputtest->tmp_filename,
                           CR_CW_GZ_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
                           FILE_COMPRESSED_1_CONTENT_LEN);
-    printf("puts()\n");
     test_helper_cw_output(OUTPUT_TYPE_PUTS,   outputtest->tmp_filename,
                           CR_CW_GZ_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
                           FILE_COMPRESSED_0_CONTENT_LEN);
     test_helper_cw_output(OUTPUT_TYPE_PUTS,   outputtest->tmp_filename,
                           CR_CW_GZ_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
                           FILE_COMPRESSED_1_CONTENT_LEN);
-    printf("printf()\n");
     test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename,
                           CR_CW_GZ_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
                           FILE_COMPRESSED_0_CONTENT_LEN);
@@ -417,21 +411,18 @@ outputtest_cw_output(Outputtest *outputtest, gconstpointer test_data)
 
     // Bz2
 
-    printf("Testing - bz2\nwrite()\n");
     test_helper_cw_output(OUTPUT_TYPE_WRITE,  outputtest->tmp_filename,
                           CR_CW_BZ2_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
                           FILE_COMPRESSED_0_CONTENT_LEN);
     test_helper_cw_output(OUTPUT_TYPE_WRITE,  outputtest->tmp_filename,
                           CR_CW_BZ2_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
                           FILE_COMPRESSED_1_CONTENT_LEN);
-    printf("puts()\n");
     test_helper_cw_output(OUTPUT_TYPE_PUTS,   outputtest->tmp_filename,
                           CR_CW_BZ2_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
                           FILE_COMPRESSED_0_CONTENT_LEN);
     test_helper_cw_output(OUTPUT_TYPE_PUTS,   outputtest->tmp_filename,
                           CR_CW_BZ2_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
                           FILE_COMPRESSED_1_CONTENT_LEN);
-    printf("printf()\n");
     test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename,
                           CR_CW_BZ2_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
                           FILE_COMPRESSED_0_CONTENT_LEN);
@@ -441,21 +432,18 @@ outputtest_cw_output(Outputtest *outputtest, gconstpointer test_data)
 
     // Xz
 
-    printf("Testing - xz\nwrite()\n");
     test_helper_cw_output(OUTPUT_TYPE_WRITE,  outputtest->tmp_filename,
                           CR_CW_XZ_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
                           FILE_COMPRESSED_0_CONTENT_LEN);
     test_helper_cw_output(OUTPUT_TYPE_WRITE,  outputtest->tmp_filename,
                           CR_CW_XZ_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
                           FILE_COMPRESSED_1_CONTENT_LEN);
-    printf("puts()\n");
     test_helper_cw_output(OUTPUT_TYPE_PUTS,   outputtest->tmp_filename,
                           CR_CW_XZ_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
                           FILE_COMPRESSED_0_CONTENT_LEN);
     test_helper_cw_output(OUTPUT_TYPE_PUTS,   outputtest->tmp_filename,
                           CR_CW_XZ_COMPRESSION, FILE_COMPRESSED_1_CONTENT,
                           FILE_COMPRESSED_1_CONTENT_LEN);
-    printf("printf()\n");
     test_helper_cw_output(OUTPUT_TYPE_PRINTF, outputtest->tmp_filename,
                           CR_CW_XZ_COMPRESSION, FILE_COMPRESSED_0_CONTENT,
                           FILE_COMPRESSED_0_CONTENT_LEN);
index 42a664c..5f2604a 100644 (file)
@@ -562,7 +562,8 @@ compressfile_test_text_file(Copyfiletest *copyfiletest, gconstpointer test_data)
 }
 
 static void
-compressfile_with_stat_test_text_file(Copyfiletest *copyfiletest, gconstpointer test_data)
+compressfile_with_stat_test_text_file(Copyfiletest *copyfiletest,
+                                      gconstpointer test_data)
 {
     CR_UNUSED(test_data);
     int ret;
@@ -587,6 +588,33 @@ compressfile_with_stat_test_text_file(Copyfiletest *copyfiletest, gconstpointer
 }
 
 static void
+decompressfile_with_stat_test_text_file(Copyfiletest *copyfiletest,
+                                        gconstpointer test_data)
+{
+    CR_UNUSED(test_data);
+    int ret;
+    char *checksum;
+    cr_ContentStat *stat;
+    GError *tmp_err = NULL;
+
+    stat = cr_contentstat_new(CR_CHECKSUM_SHA256, &tmp_err);
+    g_assert(stat);
+    g_assert(!tmp_err);
+
+    g_assert(!g_file_test(copyfiletest->dst_file, G_FILE_TEST_EXISTS));
+    ret = cr_decompress_file_with_stat(TEST_TEXT_FILE_GZ, copyfiletest->dst_file,
+                                       CR_CW_GZ_COMPRESSION, stat, &tmp_err);
+    g_assert(!tmp_err);
+    g_assert_cmpint(ret, ==, CRE_OK);
+    g_assert(g_file_test(copyfiletest->dst_file, G_FILE_TEST_IS_REGULAR));
+    g_assert_cmpstr(stat->checksum, ==, TEST_TEXT_FILE_SHA256SUM);
+    cr_contentstat_free(stat, &tmp_err);
+    g_assert(!tmp_err);
+}
+
+
+
+static void
 test_cr_download_valid_url_1(Copyfiletest *copyfiletest, gconstpointer test_data)
 {
     CR_UNUSED(test_data);
@@ -1068,6 +1096,9 @@ main(int argc, char *argv[])
     g_test_add("/misc/compressfile_with_stat_test_text_file",
             Copyfiletest, NULL, copyfiletest_setup,
             compressfile_with_stat_test_text_file, copyfiletest_teardown);
+    g_test_add("/misc/decompressfile_with_stat_test_text_file",
+            Copyfiletest, NULL, copyfiletest_setup,
+            decompressfile_with_stat_test_text_file, copyfiletest_teardown);
 //    g_test_add("/misc/test_cr_download_valid_url_1",
 //          Copyfiletest, NULL, copyfiletest_setup,
 //          test_cr_download_valid_url_1, copyfiletest_teardown);
index ec0bc4e..421f8e4 100644 (file)
@@ -236,8 +236,8 @@ test_cr_db_add_primary_pkg(TestData *testdata, gconstpointer test_data)
     cr_db_close(db, &err);
     tclean = g_timer_elapsed(timer, NULL) - tmp;
 
-    printf("Stats:\nOpen:    %f\nAdd:     %f\nCleanup: %f\nSum:     %f\n",
-           topen, tadd, tclean, (tadd + tclean));
+    //printf("Stats:\nOpen:    %f\nAdd:     %f\nCleanup: %f\nSum:     %f\n",
+    //       topen, tadd, tclean, (tadd + tclean));
 
     // Cleanup
 
diff --git a/tests/testdata/test_files/text_file.gz b/tests/testdata/test_files/text_file.gz
new file mode 100644 (file)
index 0000000..cd2bb13
Binary files /dev/null and b/tests/testdata/test_files/text_file.gz differ