From: Markus Lehtonen Date: Thu, 22 Aug 2013 22:04:56 +0000 (+0300) Subject: Fix posix tar generation X-Git-Tag: tizen/1.30-20140227~7 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=a74d8907fb3b25f4da14e6e9db061f7f0c36c550;p=tools%2Fpristine-tar.git Fix posix tar generation Remove all timestamps the PAX extended headers of posix tar archives. Change-Id: I5db7d76be5256442d10b6ed1d35fe2b868dd693c Signed-off-by: Markus Lehtonen --- diff --git a/debian/changelog b/debian/changelog index 8e9e43f..13c2c7f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +pristine-tar (1.28-2) unstable; urgency=low + + * Remove all timestamps from extended PAX headers of posix tarballs + + -- Markus Lehtonen Fri, 23 Aug 2013 11:28:41 +0300 + pristine-tar (1.28-1) unstable; urgency=low * Fixes Bug #998 diff --git a/debian/patches/0001-pristine-gz-obey-the-XDELTA_PROGRAM-build-parameter.patch b/debian/patches/0001-pristine-gz-obey-the-XDELTA_PROGRAM-build-parameter.patch new file mode 100644 index 0000000..b3bbab9 --- /dev/null +++ b/debian/patches/0001-pristine-gz-obey-the-XDELTA_PROGRAM-build-parameter.patch @@ -0,0 +1,44 @@ +From: Markus Lehtonen +Date: Fri, 31 May 2013 11:01:05 +0300 +Subject: pristine-gz: obey the XDELTA_PROGRAM build parameter + +Change-Id: I571466bacc8b50a7b2d2c7681b4d7ab1083508ad +Signed-off-by: Markus Lehtonen +--- + pristine-gz | 9 +++++++-- + 1 file changed, 7 insertions(+), 2 deletions(-) + +diff --git a/pristine-gz b/pristine-gz +index 3668256..00c0055 100755 +--- a/pristine-gz ++++ b/pristine-gz +@@ -96,6 +96,11 @@ use File::Basename qw/basename/; + + delete $ENV{GZIP}; + ++# The following assignment is potentially munged during the ++# build process to hold the value of XDELTA_PROGRAM ++# parameter as given to Makefile.PL. ++my $xdelta_program = "xdelta"; ++ + dispatch( + commands => { + usage => [\&usage], +@@ -234,7 +239,7 @@ sub reproducegz { + else { + # generate a binary delta and see if this is the + # best variant so far +- my $ret=system("xdelta delta -0 --pristine $tempout $orig $tempdir/tmpdelta 2>/dev/null") >> 8; ++ my $ret=system("$xdelta_program delta -0 --pristine $tempout $orig $tempdir/tmpdelta 2>/dev/null") >> 8; + # xdelta exits 1 on success + if ($ret == 1) { + my $size=(stat("$tempdir/tmpdelta"))[7]; +@@ -297,7 +302,7 @@ sub gengz { + my $tempdir=tempdir(); + my $tfile="$tempdir/".basename($file).".gz"; + doit_redir($file, $tfile, @zgz); +- doit("xdelta", "patch", "--pristine", $delta->{delta}, $tfile, "$file.gz"); ++ doit("$xdelta_program", "patch", "--pristine", $delta->{delta}, $tfile, "$file.gz"); + } + else { + doit_redir("$file", "$file.gz", @zgz); diff --git a/debian/patches/0002-implement-tar-format-guessing.patch b/debian/patches/0002-implement-tar-format-guessing.patch new file mode 100644 index 0000000..028d02b --- /dev/null +++ b/debian/patches/0002-implement-tar-format-guessing.patch @@ -0,0 +1,60 @@ +From: Markus Lehtonen +Date: Fri, 31 May 2013 12:45:51 +0300 +Subject: implement tar format guessing + +Try "posix" and "gnu" formats if tarball re-creation with the default +tar format fails. + +Makes pristine-tar more compatible accross different Linux distributions +as the default format used by tar varies. + +Change-Id: I3c345a632f8b02b723a6151b08dc36521fe5646b +Signed-off-by: Markus Lehtonen +--- + pristine-tar | 19 ++++++++++++++----- + 1 file changed, 14 insertions(+), 5 deletions(-) + +diff --git a/pristine-tar b/pristine-tar +index 14bac15..1c321d4 100755 +--- a/pristine-tar ++++ b/pristine-tar +@@ -343,18 +343,23 @@ sub recreatetarball { + + delete $ENV{TAR_LONGLINK_100}; + $recreatetarball_tempdir=$tempdir; +- return recreatetarball_helper(); ++ return recreatetarball_helper(%options); + } + + sub recreatetarball_helper { ++ my %options=@_; + my $tempdir=$recreatetarball_tempdir; + + my $ret="$tempdir/recreatetarball"; ++ my @cmd=($tar_program, "cf", $ret, "--owner", 0, "--group", 0, ++ "--numeric-owner", "-C", "$tempdir/workdir", ++ "--no-recursion", "--mode", "0644", ++ "--files-from", "$tempdir/manifest"); ++ if (exists $options{tar_format}) { ++ push @cmd, ("-H", $options{tar_format}); ++ } + +- doit($tar_program, "cf", $ret, "--owner", 0, "--group", 0, +- "--numeric-owner", "-C", "$tempdir/workdir", +- "--no-recursion", "--mode", "0644", +- "--files-from", "$tempdir/manifest"); ++ doit(@cmd); + + return $ret; + } +@@ -393,6 +398,10 @@ sub gentar { + push @try, sub { recreatetarball($delta->{manifest}, getcwd, + clobber_source => 0, %opts) }; + push @try, \&recreatetarball_longlink_100; ++ push @try, sub { recreatetarball($delta->{manifest}, getcwd, ++ clobber_source => 0, tar_format => "gnu", %opts) }; ++ push @try, sub { recreatetarball($delta->{manifest}, getcwd, ++ clobber_source => 0, tar_format => "posix", %opts) }; + + my $ok; + foreach my $variant (@try) { diff --git a/debian/patches/0003-Use-posix-tar-format-by-default.patch b/debian/patches/0003-Use-posix-tar-format-by-default.patch new file mode 100644 index 0000000..cf712c1 --- /dev/null +++ b/debian/patches/0003-Use-posix-tar-format-by-default.patch @@ -0,0 +1,54 @@ +From: Markus Lehtonen +Date: Fri, 31 May 2013 13:51:40 +0300 +Subject: Use posix tar format by default + +Should lead to smaller deltas as e.g. git-archive produces tarballs in +posix format. + +The tar format guessing makes sure that pristine-tar commits generated +with "gnu" tar format also are usable. + +Change-Id: I52c20b81c4b02ec22d3744d49b64415af14e0ab4 +Signed-off-by: Markus Lehtonen +--- + pristine-tar | 11 +++++------ + 1 file changed, 5 insertions(+), 6 deletions(-) + +diff --git a/pristine-tar b/pristine-tar +index 1c321d4..f1fae98 100755 +--- a/pristine-tar ++++ b/pristine-tar +@@ -396,12 +396,10 @@ sub gentar { + + my @try; + push @try, sub { recreatetarball($delta->{manifest}, getcwd, +- clobber_source => 0, %opts) }; +- push @try, \&recreatetarball_longlink_100; ++ clobber_source => 0, tar_format => "posix", %opts) }; + push @try, sub { recreatetarball($delta->{manifest}, getcwd, + clobber_source => 0, tar_format => "gnu", %opts) }; +- push @try, sub { recreatetarball($delta->{manifest}, getcwd, +- clobber_source => 0, tar_format => "posix", %opts) }; ++ push @try, \&recreatetarball_longlink_100; + + my $ok; + foreach my $variant (@try) { +@@ -511,7 +509,8 @@ sub gendelta { + if ($#out == 0 && -d $out[0]) { + $sourcedir=$out[0]; + } +- $recreatetarball=recreatetarball("$tempdir/manifest", $sourcedir, clobber_source => 1); ++ $recreatetarball=recreatetarball("$tempdir/manifest", $sourcedir, ++ clobber_source => 1, tar_format => "posix"); + } + else { + $recreatetarball=$opts{recreatetarball}; +@@ -778,7 +777,7 @@ sub commit { + my ($sourcedir, $id)=export($upstream); + genmanifest($tarball, "$tempdir/manifest"); + my $recreatetarball=recreatetarball("$tempdir/manifest", $sourcedir, +- clobber_source => 1, create_missing => 1); ++ clobber_source => 1, create_missing => 1, tar_format => "posix"); + my $pid = open(GENDELTA, "-|"); + if (! $pid) { + # child diff --git a/debian/patches/0004-Mangle-PAX-headers-when-using-posix-tar-format.patch b/debian/patches/0004-Mangle-PAX-headers-when-using-posix-tar-format.patch new file mode 100644 index 0000000..bb2b418 --- /dev/null +++ b/debian/patches/0004-Mangle-PAX-headers-when-using-posix-tar-format.patch @@ -0,0 +1,28 @@ +From: Markus Lehtonen +Date: Thu, 6 Jun 2013 14:31:25 +0300 +Subject: Mangle PAX headers when using posix tar format + +Mangle tar PAX headers in order to guarantee that created tarballs are +always identical so that the delta can be applied cleanly. + +Signed-off-by: Markus Lehtonen +--- + pristine-tar | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/pristine-tar b/pristine-tar +index f1fae98..3e92a74 100755 +--- a/pristine-tar ++++ b/pristine-tar +@@ -357,6 +357,11 @@ sub recreatetarball_helper { + "--files-from", "$tempdir/manifest"); + if (exists $options{tar_format}) { + push @cmd, ("-H", $options{tar_format}); ++ if ($options{tar_format} eq 'posix') { ++ # Mangle PAX headers so that created tarballs are always identical ++ # so that delta can be successfully applied ++ push @cmd, ('--pax-option=exthdr.name=%d/PaxHeaders/%f,ctime:=0'); ++ } + } + + doit(@cmd); diff --git a/debian/patches/0005-Remove-all-timestamps-from-extended-PAX-headers.patch b/debian/patches/0005-Remove-all-timestamps-from-extended-PAX-headers.patch new file mode 100644 index 0000000..4d5cbe2 --- /dev/null +++ b/debian/patches/0005-Remove-all-timestamps-from-extended-PAX-headers.patch @@ -0,0 +1,28 @@ +From: Markus Lehtonen +Date: Fri, 23 Aug 2013 00:47:12 +0300 +Subject: Remove all timestamps from extended PAX headers + +The timestamps can break pristine-tar generation as they are different +every time an archive is generated from git (when using tree id +instead of commit id as the sha-1 to checkout). Git uses the current +time as the modification time of files when a tree is archived. + +Change-Id: I25fc631c4f1c62fc5ec2c1a8112dd01c643aae90 +Signed-off-by: Markus Lehtonen +--- + pristine-tar | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/pristine-tar b/pristine-tar +index 3e92a74..19af965 100755 +--- a/pristine-tar ++++ b/pristine-tar +@@ -360,7 +360,7 @@ sub recreatetarball_helper { + if ($options{tar_format} eq 'posix') { + # Mangle PAX headers so that created tarballs are always identical + # so that delta can be successfully applied +- push @cmd, ('--pax-option=exthdr.name=%d/PaxHeaders/%f,ctime:=0'); ++ push @cmd, ('--pax-option=exthdr.name=%d/PaxHeaders/%f,mtime:=0,atime:=0,ctime:=0'); + } + } + diff --git a/debian/patches/0005-pristine-gz-obey-the-XDELTA_PROGRAM-build-parameter.patch b/debian/patches/0005-pristine-gz-obey-the-XDELTA_PROGRAM-build-parameter.patch deleted file mode 100644 index b3bbab9..0000000 --- a/debian/patches/0005-pristine-gz-obey-the-XDELTA_PROGRAM-build-parameter.patch +++ /dev/null @@ -1,44 +0,0 @@ -From: Markus Lehtonen -Date: Fri, 31 May 2013 11:01:05 +0300 -Subject: pristine-gz: obey the XDELTA_PROGRAM build parameter - -Change-Id: I571466bacc8b50a7b2d2c7681b4d7ab1083508ad -Signed-off-by: Markus Lehtonen ---- - pristine-gz | 9 +++++++-- - 1 file changed, 7 insertions(+), 2 deletions(-) - -diff --git a/pristine-gz b/pristine-gz -index 3668256..00c0055 100755 ---- a/pristine-gz -+++ b/pristine-gz -@@ -96,6 +96,11 @@ use File::Basename qw/basename/; - - delete $ENV{GZIP}; - -+# The following assignment is potentially munged during the -+# build process to hold the value of XDELTA_PROGRAM -+# parameter as given to Makefile.PL. -+my $xdelta_program = "xdelta"; -+ - dispatch( - commands => { - usage => [\&usage], -@@ -234,7 +239,7 @@ sub reproducegz { - else { - # generate a binary delta and see if this is the - # best variant so far -- my $ret=system("xdelta delta -0 --pristine $tempout $orig $tempdir/tmpdelta 2>/dev/null") >> 8; -+ my $ret=system("$xdelta_program delta -0 --pristine $tempout $orig $tempdir/tmpdelta 2>/dev/null") >> 8; - # xdelta exits 1 on success - if ($ret == 1) { - my $size=(stat("$tempdir/tmpdelta"))[7]; -@@ -297,7 +302,7 @@ sub gengz { - my $tempdir=tempdir(); - my $tfile="$tempdir/".basename($file).".gz"; - doit_redir($file, $tfile, @zgz); -- doit("xdelta", "patch", "--pristine", $delta->{delta}, $tfile, "$file.gz"); -+ doit("$xdelta_program", "patch", "--pristine", $delta->{delta}, $tfile, "$file.gz"); - } - else { - doit_redir("$file", "$file.gz", @zgz); diff --git a/debian/patches/0007-implement-tar-format-guessing.patch b/debian/patches/0007-implement-tar-format-guessing.patch deleted file mode 100644 index 028d02b..0000000 --- a/debian/patches/0007-implement-tar-format-guessing.patch +++ /dev/null @@ -1,60 +0,0 @@ -From: Markus Lehtonen -Date: Fri, 31 May 2013 12:45:51 +0300 -Subject: implement tar format guessing - -Try "posix" and "gnu" formats if tarball re-creation with the default -tar format fails. - -Makes pristine-tar more compatible accross different Linux distributions -as the default format used by tar varies. - -Change-Id: I3c345a632f8b02b723a6151b08dc36521fe5646b -Signed-off-by: Markus Lehtonen ---- - pristine-tar | 19 ++++++++++++++----- - 1 file changed, 14 insertions(+), 5 deletions(-) - -diff --git a/pristine-tar b/pristine-tar -index 14bac15..1c321d4 100755 ---- a/pristine-tar -+++ b/pristine-tar -@@ -343,18 +343,23 @@ sub recreatetarball { - - delete $ENV{TAR_LONGLINK_100}; - $recreatetarball_tempdir=$tempdir; -- return recreatetarball_helper(); -+ return recreatetarball_helper(%options); - } - - sub recreatetarball_helper { -+ my %options=@_; - my $tempdir=$recreatetarball_tempdir; - - my $ret="$tempdir/recreatetarball"; -+ my @cmd=($tar_program, "cf", $ret, "--owner", 0, "--group", 0, -+ "--numeric-owner", "-C", "$tempdir/workdir", -+ "--no-recursion", "--mode", "0644", -+ "--files-from", "$tempdir/manifest"); -+ if (exists $options{tar_format}) { -+ push @cmd, ("-H", $options{tar_format}); -+ } - -- doit($tar_program, "cf", $ret, "--owner", 0, "--group", 0, -- "--numeric-owner", "-C", "$tempdir/workdir", -- "--no-recursion", "--mode", "0644", -- "--files-from", "$tempdir/manifest"); -+ doit(@cmd); - - return $ret; - } -@@ -393,6 +398,10 @@ sub gentar { - push @try, sub { recreatetarball($delta->{manifest}, getcwd, - clobber_source => 0, %opts) }; - push @try, \&recreatetarball_longlink_100; -+ push @try, sub { recreatetarball($delta->{manifest}, getcwd, -+ clobber_source => 0, tar_format => "gnu", %opts) }; -+ push @try, sub { recreatetarball($delta->{manifest}, getcwd, -+ clobber_source => 0, tar_format => "posix", %opts) }; - - my $ok; - foreach my $variant (@try) { diff --git a/debian/patches/0008-Use-posix-tar-format-by-default.patch b/debian/patches/0008-Use-posix-tar-format-by-default.patch deleted file mode 100644 index cf712c1..0000000 --- a/debian/patches/0008-Use-posix-tar-format-by-default.patch +++ /dev/null @@ -1,54 +0,0 @@ -From: Markus Lehtonen -Date: Fri, 31 May 2013 13:51:40 +0300 -Subject: Use posix tar format by default - -Should lead to smaller deltas as e.g. git-archive produces tarballs in -posix format. - -The tar format guessing makes sure that pristine-tar commits generated -with "gnu" tar format also are usable. - -Change-Id: I52c20b81c4b02ec22d3744d49b64415af14e0ab4 -Signed-off-by: Markus Lehtonen ---- - pristine-tar | 11 +++++------ - 1 file changed, 5 insertions(+), 6 deletions(-) - -diff --git a/pristine-tar b/pristine-tar -index 1c321d4..f1fae98 100755 ---- a/pristine-tar -+++ b/pristine-tar -@@ -396,12 +396,10 @@ sub gentar { - - my @try; - push @try, sub { recreatetarball($delta->{manifest}, getcwd, -- clobber_source => 0, %opts) }; -- push @try, \&recreatetarball_longlink_100; -+ clobber_source => 0, tar_format => "posix", %opts) }; - push @try, sub { recreatetarball($delta->{manifest}, getcwd, - clobber_source => 0, tar_format => "gnu", %opts) }; -- push @try, sub { recreatetarball($delta->{manifest}, getcwd, -- clobber_source => 0, tar_format => "posix", %opts) }; -+ push @try, \&recreatetarball_longlink_100; - - my $ok; - foreach my $variant (@try) { -@@ -511,7 +509,8 @@ sub gendelta { - if ($#out == 0 && -d $out[0]) { - $sourcedir=$out[0]; - } -- $recreatetarball=recreatetarball("$tempdir/manifest", $sourcedir, clobber_source => 1); -+ $recreatetarball=recreatetarball("$tempdir/manifest", $sourcedir, -+ clobber_source => 1, tar_format => "posix"); - } - else { - $recreatetarball=$opts{recreatetarball}; -@@ -778,7 +777,7 @@ sub commit { - my ($sourcedir, $id)=export($upstream); - genmanifest($tarball, "$tempdir/manifest"); - my $recreatetarball=recreatetarball("$tempdir/manifest", $sourcedir, -- clobber_source => 1, create_missing => 1); -+ clobber_source => 1, create_missing => 1, tar_format => "posix"); - my $pid = open(GENDELTA, "-|"); - if (! $pid) { - # child diff --git a/debian/patches/0009-Mangle-PAX-headers-when-using-posix-tar-format.patch b/debian/patches/0009-Mangle-PAX-headers-when-using-posix-tar-format.patch deleted file mode 100644 index bb2b418..0000000 --- a/debian/patches/0009-Mangle-PAX-headers-when-using-posix-tar-format.patch +++ /dev/null @@ -1,28 +0,0 @@ -From: Markus Lehtonen -Date: Thu, 6 Jun 2013 14:31:25 +0300 -Subject: Mangle PAX headers when using posix tar format - -Mangle tar PAX headers in order to guarantee that created tarballs are -always identical so that the delta can be applied cleanly. - -Signed-off-by: Markus Lehtonen ---- - pristine-tar | 5 +++++ - 1 file changed, 5 insertions(+) - -diff --git a/pristine-tar b/pristine-tar -index f1fae98..3e92a74 100755 ---- a/pristine-tar -+++ b/pristine-tar -@@ -357,6 +357,11 @@ sub recreatetarball_helper { - "--files-from", "$tempdir/manifest"); - if (exists $options{tar_format}) { - push @cmd, ("-H", $options{tar_format}); -+ if ($options{tar_format} eq 'posix') { -+ # Mangle PAX headers so that created tarballs are always identical -+ # so that delta can be successfully applied -+ push @cmd, ('--pax-option=exthdr.name=%d/PaxHeaders/%f,ctime:=0'); -+ } - } - - doit(@cmd); diff --git a/debian/patches/series b/debian/patches/series index c034b74..7e32c85 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -1,4 +1,5 @@ -0005-pristine-gz-obey-the-XDELTA_PROGRAM-build-parameter.patch -0007-implement-tar-format-guessing.patch -0008-Use-posix-tar-format-by-default.patch -0009-Mangle-PAX-headers-when-using-posix-tar-format.patch +0001-pristine-gz-obey-the-XDELTA_PROGRAM-build-parameter.patch +0002-implement-tar-format-guessing.patch +0003-Use-posix-tar-format-by-default.patch +0004-Mangle-PAX-headers-when-using-posix-tar-format.patch +0005-Remove-all-timestamps-from-extended-PAX-headers.patch diff --git a/packaging/0010-Remove-all-timestamps-from-extended-PAX-headers.patch b/packaging/0010-Remove-all-timestamps-from-extended-PAX-headers.patch new file mode 100644 index 0000000..4d5cbe2 --- /dev/null +++ b/packaging/0010-Remove-all-timestamps-from-extended-PAX-headers.patch @@ -0,0 +1,28 @@ +From: Markus Lehtonen +Date: Fri, 23 Aug 2013 00:47:12 +0300 +Subject: Remove all timestamps from extended PAX headers + +The timestamps can break pristine-tar generation as they are different +every time an archive is generated from git (when using tree id +instead of commit id as the sha-1 to checkout). Git uses the current +time as the modification time of files when a tree is archived. + +Change-Id: I25fc631c4f1c62fc5ec2c1a8112dd01c643aae90 +Signed-off-by: Markus Lehtonen +--- + pristine-tar | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/pristine-tar b/pristine-tar +index 3e92a74..19af965 100755 +--- a/pristine-tar ++++ b/pristine-tar +@@ -360,7 +360,7 @@ sub recreatetarball_helper { + if ($options{tar_format} eq 'posix') { + # Mangle PAX headers so that created tarballs are always identical + # so that delta can be successfully applied +- push @cmd, ('--pax-option=exthdr.name=%d/PaxHeaders/%f,ctime:=0'); ++ push @cmd, ('--pax-option=exthdr.name=%d/PaxHeaders/%f,mtime:=0,atime:=0,ctime:=0'); + } + } + diff --git a/packaging/pristine-tar.changes b/packaging/pristine-tar.changes index 66bfbb5..1b6ad75 100644 --- a/packaging/pristine-tar.changes +++ b/packaging/pristine-tar.changes @@ -1,3 +1,6 @@ +* Fri Aug 23 2013 Markus Lehtonen 1.28 +- Remove all timestamps from extended PAX headers of posix tarballs + * Mon Jun 03 2013 Markus Lehtonen 1.28 - Fixes Bug #998 * Mangle PAX headers when using posix tar format diff --git a/packaging/pristine-tar.spec b/packaging/pristine-tar.spec index 0f72e66..f3e83d9 100644 --- a/packaging/pristine-tar.spec +++ b/packaging/pristine-tar.spec @@ -16,6 +16,7 @@ Patch5: 0006-Add-.gbp.conf.patch Patch6: 0007-implement-tar-format-guessing.patch Patch7: 0008-Use-posix-tar-format-by-default.patch Patch8: 0009-Mangle-PAX-headers-when-using-posix-tar-format.patch +Patch9: 0010-Remove-all-timestamps-from-extended-PAX-headers.patch Requires: gzip Requires: bzip2 %if 0%{?suse_version} >= 1210 @@ -89,6 +90,8 @@ control. %patch7 -p1 # 0009-Mangle-PAX-headers-when-using-posix-tar-format.patch %patch8 -p1 +# 0010-Remove-all-timestamps-from-extended-PAX-headers.patch +%patch9 -p1 %build %if 0%{?fedora}