import-orig: Allow the post-unpack hook to filter tarball files

With post-unpack scripts is currently possible to filter out files that are
not needed from the upstream branch, however it is not possible to use it to
filter files that will end up in the orig file, and in some scenarios this
is not easily doable just using a filter list.

So, run the post-unpack hook just after unpacking and before repacking the
tar.

Add tests to verify this both when using with regular --filter option
and by itself.

Closes: #812721

diff --git a/gbp/scripts/common/import_orig.py b/gbp/scripts/common/import_orig.py
index 59093683db8aed55f182596a4a0eb35c1347255d..8b573095c8853916a58edbfc78881c29fcbb9b31 100644 (file)
--- a/gbp/scripts/common/import_orig.py
+++ b/gbp/scripts/common/import_orig.py
@@ -40,20 +40,26 @@ def orig_needs_repack(upstream_source, options):
     Determine if the upstream sources needs to be repacked
 
     We repack if
-     1. we want to filter out files and use pristine tar since we want
-        to make a filtered tarball available to pristine-tar
+     1. we want to filter out files via filters or post-unpack script and use
+        pristine tar since we want to make a filtered tarball available to
+        pristine-tar
      2. we don't have a suitable upstream tarball (e.g. zip archive or unpacked dir)
         and want to use filters
      3. we don't have a suitable upstream tarball (e.g. zip archive or unpacked dir)
         and want to use pristine-tar
+     4. we don't have a suitable upstream tarball (e.g. zip archive or unpacked dir)
+        and want to use a post-unpack script
     """
-    if ((options.pristine_tar and options.filter_pristine_tar and len(options.filters) > 0)):
+    if ((options.pristine_tar and options.filter_pristine_tar and
+       (options.filters or options.postunpack))):
         return True
     elif not upstream_source.is_orig():
         if len(options.filters):
             return True
         elif options.pristine_tar:
             return True
+        elif options.postunpack:
+            return True
     return False
 
 

diff --git a/gbp/scripts/import_orig.py b/gbp/scripts/import_orig.py
index 2947235a020ab3e1b6ef954ea3b5eacf42920dd9..215bc8ada917dbdb3fc53518732576b8afc75f94 100644 (file)
--- a/gbp/scripts/import_orig.py
+++ b/gbp/scripts/import_orig.py
@@ -293,12 +293,17 @@ def debian_branch_merge_by_merge(repo, tag, version, options):
     repo.set_branch(branch)
 
 
-def unpack_tarballs(name, sources, version, options):
+def unpack_tarballs(repo, name, sources, version, options):
     tmpdir = tempfile.mkdtemp(dir='../')
     if not sources[0].is_dir():  # Unpack main tarball
         sources[0].unpack(tmpdir, options.filters)
         gbp.log.debug("Unpacked '%s' to '%s'" % (sources[0].path, sources[0].unpacked))
 
+    try:
+        postunpack_hook(repo, tmpdir, sources, options)
+    except gbpc.CommandExecFailed:
+        raise GbpError()  # The hook already printed an error message
+
     if orig_needs_repack(sources[0], options):
         gbp.log.debug("Filter pristine-tar: repacking '%s' from '%s'" % (sources[0].path,
                                                                          sources[0].unpacked))
@@ -473,11 +478,7 @@ def main(argv):
         if repo.bare:
             set_bare_repo_options(options)
 
-        sources, tmpdir = unpack_tarballs(name, sources, version, options)
-        try:
-            postunpack_hook(repo, tmpdir, sources, options)
-        except gbpc.CommandExecFailed:
-            raise GbpError()  # The hook already printed an error message
+        sources, tmpdir = unpack_tarballs(repo, name, sources, version, options)
 
         if options.verbose:
             for source in sources:

diff --git a/tests/component/deb/test_import_orig.py b/tests/component/deb/test_import_orig.py
index ef75919e2f57b1ccbbb2d00098a66148943f955e..cadda636cab934e3435bf48b81bea50336c08314 100644 (file)
--- a/tests/component/deb/test_import_orig.py
+++ b/tests/component/deb/test_import_orig.py
@@ -273,6 +273,49 @@ class TestImportOrig(ComponentTestBase):
                 t.getmember(f)
         t.close()
 
+    @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
+    def test_filter_with_component_tarballs_and_postunpack_changes(self, repo):
+        """
+        Test that using a filter works with component tarballs (#840602) and
+        that the postunpack hook can be used to do more sophisticated changes
+        to the orig (#951534).
+        """
+        # copy data since we don't want the repacked tarball to end up in DEB_TEST_DATA_DIR
+        os.mkdir('../tarballs')
+        for f in ['hello-debhelper_2.8.orig-foo.tar.gz', 'hello-debhelper_2.8.orig.tar.gz']:
+            src = os.path.join(DEB_TEST_DATA_DIR, 'dsc-3.0-additional-tarballs', f)
+            shutil.copy(src, '../tarballs')
+
+        ok_(import_orig(['arg0',
+                         '--component=foo',
+                         '--no-interactive',
+                         '--pristine-tar',
+                         '--filter-pristine-tar',
+                         '--filter=README*',
+                         '--postunpack=printenv > $GBP_SOURCES_DIR/postunpack.out;' +
+                         'rm $GBP_SOURCES_DIR/TODO',
+                         '../tarballs/hello-debhelper_2.8.orig.tar.gz']) == 0)
+        self._check_repo_state(repo, 'master', ['master', 'upstream', 'pristine-tar'],
+                               tags=['debian/2.6-2', 'upstream/2.6', 'upstream/2.8'])
+        self._check_component_tarballs(repo, ['foo/test1', 'foo/test2'])
+
+        ok_('README' not in repo.ls_tree('HEAD'),
+            "README not filtered out of %s" % repo.ls_tree('HEAD'))
+        ok_('TODO' not in repo.ls_tree('HEAD'),
+            "TODO not filtered out of %s" % repo.ls_tree('HEAD'))
+        tar = '../hello-debhelper_2.8.orig.tar.gz'
+
+        # Check if tar got filtered properly
+        ok_(os.path.exists(tar))
+        t = tarfile.open(name=tar, mode="r:gz")
+        for f in ['hello-2.8/configure', 'hello-2.8/postunpack.out']:
+            i = t.getmember(f)
+            eq_(type(i), tarfile.TarInfo)
+        for f in ['hello-2.8/README', 'hello-2.8/TODO']:
+            with assert_raises(KeyError):
+                t.getmember(f)
+        t.close()
+
     @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
     def test_filter_with_orig_tarball(self, repo):
         """
@@ -306,6 +349,76 @@ class TestImportOrig(ComponentTestBase):
                 t.getmember(f)
         t.close()
 
+    @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
+    def test_filter_with_orig_tarball_and_postunpack_changes(self, repo):
+        """
+        Test that using a filter works with an upstream tarball that has
+        already the correct name (#558777) and that the postunpack hook can
+        be used to do more sophisticated changes to the orig (#951534).
+        """
+        f = 'hello-debhelper_2.8.orig.tar.gz'
+        src = os.path.join(DEB_TEST_DATA_DIR, 'dsc-3.0', f)
+        shutil.copy(src, '..')
+
+        ok_(import_orig(['arg0',
+                         '--no-interactive',
+                         '--pristine-tar',
+                         '--filter-pristine-tar',
+                         '--filter=README*',
+                         '--postunpack=printenv > $GBP_SOURCES_DIR/postunpack.out;' +
+                         'rm $GBP_SOURCES_DIR/TODO',
+                         '../hello-debhelper_2.8.orig.tar.gz']) == 0)
+        self._check_repo_state(repo, 'master', ['master', 'upstream', 'pristine-tar'],
+                               tags=['debian/2.6-2', 'upstream/2.6', 'upstream/2.8'])
+
+        filtered = os.path.join('..', f)
+        ok_(os.path.exists(filtered))
+        eq_(os.readlink(filtered).split('/')[-1],
+            'hello-debhelper_2.8.orig.gbp.tar.gz')
+        # Check if tar got filtered properly
+        t = tarfile.open(name=filtered, mode="r:gz")
+        for f in ['hello-2.8/configure', 'hello-2.8/postunpack.out']:
+            i = t.getmember(f)
+            eq_(type(i), tarfile.TarInfo)
+        for f in ['hello-2.8/README', 'hello-2.8/TODO']:
+            with assert_raises(KeyError):
+                t.getmember(f)
+        t.close()
+
+    @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
+    def test_postunpack_changes_with_orig_tarball(self, repo):
+        """
+        Test that using a postupack script to apply changes works with an
+        upstream tarball that has already the correct name (#951534).
+        """
+        f = 'hello-debhelper_2.8.orig.tar.gz'
+        src = os.path.join(DEB_TEST_DATA_DIR, 'dsc-3.0', f)
+        shutil.copy(src, '..')
+
+        ok_(import_orig(['arg0',
+                         '--no-interactive',
+                         '--pristine-tar',
+                         '--filter-pristine-tar',
+                         '--postunpack=printenv > $GBP_SOURCES_DIR/postunpack.out;' +
+                         'rm $GBP_SOURCES_DIR/TODO; rm $GBP_SOURCES_DIR/README',
+                         '../hello-debhelper_2.8.orig.tar.gz']) == 0)
+        self._check_repo_state(repo, 'master', ['master', 'upstream', 'pristine-tar'],
+                               tags=['debian/2.6-2', 'upstream/2.6', 'upstream/2.8'])
+
+        filtered = os.path.join('..', f)
+        ok_(os.path.exists(filtered))
+        eq_(os.readlink(filtered).split('/')[-1],
+            'hello-debhelper_2.8.orig.gbp.tar.gz')
+        # Check if tar got filtered properly
+        t = tarfile.open(name=filtered, mode="r:gz")
+        for f in ['hello-2.8/configure', 'hello-2.8/postunpack.out']:
+            i = t.getmember(f)
+            eq_(type(i), tarfile.TarInfo)
+        for f in ['hello-2.8/README', 'hello-2.8/TODO']:
+            with assert_raises(KeyError):
+                t.getmember(f)
+        t.close()
+
     @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
     def test_filter_unpacked_dir(self, repo):
         """
@@ -339,6 +452,43 @@ class TestImportOrig(ComponentTestBase):
                 t.getmember(f)
         t.close()
 
+    @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
+    def test_filter_unpacked_dir_with_postunpack_changes(self, repo):
+        """
+        Test that importing and filtering unpacked upstream source works and
+        that the postunpack hook can be used to do more sophisticated changes
+        to the orig (#951534).
+        """
+        f = 'hello-debhelper_2.8.orig.tar.gz'
+        src = os.path.join(DEB_TEST_DATA_DIR, 'dsc-3.0', f)
+
+        # Create an unpacked tarball we can import
+        UnpackTarArchive(src, '..')()
+        ok_(os.path.exists('../hello-2.8'))
+
+        ok_(import_orig(['arg0',
+                         '--no-interactive',
+                         '--pristine-tar',
+                         '--filter-pristine-tar',
+                         '--filter=README*',
+                         '--postunpack=printenv > $GBP_SOURCES_DIR/postunpack.out;' +
+                         'rm $GBP_SOURCES_DIR/TODO',
+                         '../hello-2.8']) == 0)
+        self._check_repo_state(repo, 'master', ['master', 'upstream', 'pristine-tar'],
+                               tags=['debian/2.6-2', 'upstream/2.6', 'upstream/2.8'])
+
+        filtered = os.path.join('..', f)
+        ok_(os.path.exists(filtered))
+        # Check if tar got filtered properly
+        t = tarfile.open(name=filtered, mode="r:gz")
+        for f in ['hello-2.8/configure', 'hello-2.8/postunpack.out']:
+            i = t.getmember(f)
+            eq_(type(i), tarfile.TarInfo)
+        for f in ['hello-2.8/README', 'hello-2.8/TODO']:
+            with assert_raises(KeyError):
+                t.getmember(f)
+        t.close()
+
     @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
     def test_import_in_submodule(self, repo):
         """