Append extra sub packages after build finished

This change does not affect the build order/deps info during the build.
With this change, need additional 8 seonds for 1,885 packages
 when calculating dependency information after build is finished.

Change-Id: Ic88226d4678a0a643b1447350ce2bd3ee4ecbb73

diff --git a/depanneur b/depanneur
index a5619caf765e13db47de3bdd03d29598d2fc7501..25273d287253d65da16875121db6ef8aceea3840 100755 (executable)
--- a/depanneur
+++ b/depanneur
@@ -6,6 +6,7 @@ use File::Spec::Functions;
 use JSON;
 use HTML::Template;
 use Time::HiRes qw ( sleep time );
+use Digest::MD5 ();
 
 # Pretreatment for adding build path to search
 BEGIN {
@@ -25,6 +26,8 @@ use File::Find ();
 use Term::ANSIColor qw(:constants);
 use File::Path;
 use File::Basename;
+use File::Path qw(mkpath rmtree);
+use File::Temp qw/ tempfile tempdir /;
 use URI;
 use POSIX ":sys_wait_h";
 use File::Glob ':glob';
@@ -72,6 +75,7 @@ use Pod::Usage;
 use File::Temp qw/ tempfile tempdir /;
 use Build;
 use Build::Rpm;
+use Build::Rpmmd;
 use BSSolv;
 use Data::Dumper;
 use File::Basename;
@@ -199,6 +203,7 @@ my $tarfile = 0; # generate tar file for dependence & reverse dependence xml fil
 my $preordered_list = ""; # List of ordered packages to support user defined build order calculation
 my $profiling = ""; # Reference profiling report location. If set reports will be generated
 my $with_submodules = 0; #didn't export sub modules source code.
+my $work_done = 0; # Whether build jobs finished
 
 GetOptions (
     "repository=s" => \@repos,
@@ -1005,6 +1010,7 @@ sub parse_packs {
     my ($config, @packs) = @_;
     my %packs = ();
     my %tmp_sub_to_main = ();
+
     foreach my $spec_ref (@packs) {
         my $spec;
         my $base;
@@ -1073,6 +1079,38 @@ sub parse_packs {
             $packs{$name}{project_base_path} = $base;
         }
     }
+
+    # Append sub package information from repodata after build finished.
+    if ($work_done == 1) {
+        my @check_repos = ("$localrepo/$dist/$arch/");
+        my %recal_deps = ();
+        %recal_deps = recalculate_repomddeps(@check_repos);
+
+        foreach my $miss_pack (keys %recal_deps) {
+            # Handle only existing package
+            if (grep $_ eq $miss_pack, (keys %packs)) {
+                my $pushed = 0;
+                my @packs_subpackages = @{$packs{$miss_pack}->{'subpacks'}};
+                my @recal_rpms = @{$recal_deps{$miss_pack}};
+
+                foreach my $miss_p (@recal_rpms) {
+                  if (!(grep $_ eq $miss_p, (@packs_subpackages))) {
+                    push(@packs_subpackages, $miss_p);
+                    $pushed = 1;
+                  }
+                }
+
+                if ($pushed == 1) {
+                    @{$packs{$miss_pack}->{subpacks}} = @packs_subpackages;
+                    foreach my $sub_p (@{$packs{$miss_pack}->{subpacks}}) {
+                        $tmp_sub_to_main{$sub_p} = $miss_pack;
+                    }
+                    %subptomainp = %tmp_sub_to_main;
+                }
+            }
+        }
+    }
+
     return %packs;
 }
 
@@ -2741,6 +2779,7 @@ sub profiling_report {
     if ($profiling eq "" || ! `which bsr`) {
         return;
     }
+
     generate_depends();
 
     my $report_command = "bsr report -a $arch --depsnumbersort -j $profiling ";
@@ -2756,6 +2795,63 @@ sub profiling_report {
     my_system("mv bsr_profiling_report $localrepo/$dist/$arch/");
 }
 
+sub recalculate_repomddeps {
+
+  if ($work_done != 1) {
+    return;
+  }
+
+  # Script copied from createrepomddeps
+  my %source_rpm_mappings = (); # source to rpms from repodata
+
+  foreach my $url (@_) {
+    $url =~ s!/*$!/!; # Add a trailing slash
+    my $dir = $url;
+    my $baseurl = $url;
+    my @packages;
+    my @primaryfiles;
+
+    if (!(-e "${dir}repodata/repomd.xml")) {
+      return;
+    }
+
+    Build::Rpmmd::parse_repomd("${dir}repodata/repomd.xml", \@primaryfiles);
+
+    @primaryfiles = grep {$_->{'type'} eq 'primary' && defined($_->{'location'})} @primaryfiles;
+    for my $f (@primaryfiles) {
+      my $u = "${dir}$f->{'location'}";
+      if ($] > 5.007) {
+        require Encode;
+        utf8::downgrade($u);
+      }
+      if ($url =~ /^(?:ftps?|https?):\/\/([^\/]*)\/?/) {
+        if (system("$INC[0]/download", "${dir}repodata/", "${baseurl}repodata/" . basename($u))) {
+          die("download failed\n");
+        }
+      }
+      my $fh;
+      open($fh, '<', $u) or die "Error opening $u: $!\n";
+      if ($u =~ /\.gz$/) {
+        use IO::Uncompress::Gunzip qw($GunzipError);
+        $fh = new IO::Uncompress::Gunzip $fh or die "Error opening $u: $GunzipError\n";
+      }
+      Build::Rpmmd::parse($fh, sub {
+        $_[0]->{'baseurl'} = $baseurl;
+        push @packages, $_[0];
+      }, 'addselfprovides' => 0);
+      close($fh);
+    }
+
+    for my $p (@packages) {
+      if (!exists($source_rpm_mappings{$p->{source}})) {
+        $source_rpm_mappings{$p->{source}} = [];
+      }
+      push @{$source_rpm_mappings{$p->{source}}}, $p->{name};
+    }
+  }
+  return %source_rpm_mappings;
+}
+
 # MAIN
 if ($depends) {
     info("start generate packages depends from: " . $package_path . " ($style)");
@@ -3249,6 +3345,8 @@ while (! $TERM) {
 while ((threads->list() > 0)) {
     sleep(1);
 }
+
+$work_done = 1;
 update_repo();
 my $build_status = build_report();
 profiling_report();