Support remote repos

diff --git a/build b/build
index e8020b1..fd65285 100755 (executable)
--- a/build
+++ b/build
@@ -142,6 +142,7 @@ Known Parameters:
              repository NAME. The repo must be refreshed with zypp
              so package meta data is available locally. With emtpy
              NAME all enabled repositories are used.
+              a url can specify a remote repo.
 
   --rpms path1:path2:...
               Specify path where to find the RPMs for the build system
@@ -157,6 +158,13 @@ Known Parameters:
   --root rootdir
               Use 'rootdir' to setup chroot environment
 
+  --cachedir cachedir
+              Use 'cachedir' to cache remote repo's packages, the
+              default cache dir is /var/cache/build, every repo
+              given by --repository corresponds to a subdir named
+              as md5sum of its repo url, forx eaxmple:
+                 /var/cache/build/3e8ea9b47808629414a0cebc33ea285e
+
   --oldpackages oldpackagesdir
               Define a directory with a former build
 
@@ -702,6 +710,10 @@ while test -n "$1"; do
        BUILD_ROOT="$ARG"
        shift
       ;;
+      *-cachedir)
+       CACHE_DIR="$ARG"
+       shift
+      ;;
       *-oldpackages)
        OLD_PACKAGES="$ARG"
        shift
@@ -867,6 +879,8 @@ while test -n "$1"; do
     esac
 done
 
+test -z "$CACHE_DIR" && CACHE_DIR=/var/cache/build
+
 if test -n "$KILL" ; then
     test -z "$SRCDIR" || usage
     if test -n "$VM_IMAGE" -a -n "$VM_SWAP" -a -n "$VM_TYPE"; then
@@ -994,7 +1008,7 @@ if test -n "$LIST_STATE" ; then
        }
        for SPECFILE in $BUILD_ROOT/usr/src/packages/SPECS/*.spec ; do : ; done
     fi
-    init_buildsystem --list-state "${definesnstuff[@]}" "${repos[@]}" $USEUSEDFORBUILD $SPECFILE $BUILD_EXTRA_PACKS
+    init_buildsystem --cachedir "$CACHE_DIR" --list-state "${definesnstuff[@]}" "${repos[@]}" $USEUSEDFORBUILD $SPECFILE $BUILD_EXTRA_PACKS
     ERR=$?
     rm -rf $BUILD_ROOT
     cleanup_and_exit $ERR
@@ -1197,7 +1211,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do
        if test "$DO_INIT" = true ; then
            # do fist stage of init_buildsystem
            rm -f $BUILD_ROOT/.build.success
-           set -- init_buildsystem --prepare "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS
+           set -- init_buildsystem --cachedir "$CACHE_DIR" --prepare "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS
            echo "$* ..."
            "$@" || cleanup_and_exit 1
            check_exit
@@ -1414,7 +1428,7 @@ for SPECFILE in "${SPECFILES[@]}" ; do
        echo "BUILD_INCARNATION=$INCARNATION" > $BUILD_ROOT/.buildenv
        CREATE_BUILD_BINARIES=
        egrep '^#[       ]*needsbinariesforbuild[       ]*$' >/dev/null <$MYSRCDIR/$SPECFILE && CREATE_BUILD_BINARIES=--create-build-binaries
-       set -- init_buildsystem "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS
+       set -- init_buildsystem --cachedir "$CACHE_DIR" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $CREATE_BUILD_BINARIES $RPMLIST "$MYSRCDIR/$SPECFILE" $ADDITIONAL_PACKS
        echo "$* ..."
        "$@" || cleanup_and_exit 1
        check_exit

diff --git a/createrepomddeps b/createrepomddeps
index 61c604a..fb2f499 100755 (executable)
--- a/createrepomddeps
+++ b/createrepomddeps
@@ -9,6 +9,11 @@ use XML::Parser;
 use Data::Dumper;
 use Getopt::Long;
 use Build::Rpm;
+use Digest::MD5 qw(md5 md5_hex md5_base64);
+use File::Path qw(mkpath rmtree);
+use File::Basename;
+use LWP::UserAgent;
+use URI;
 Getopt::Long::Configure("no_ignore_case");
 
 my @parent = [];
@@ -342,6 +347,16 @@ sub printasbuildcachefile(@)
   }
 }
 
+sub getmetadata
+{
+  my $url = $_[0];
+  my $dir = $_[1];
+
+  my $dest = $dir . "repodata";
+  mkpath($dest);
+  system($INC[0].'/download', $dest, $url . "repodata/repomd.xml");
+}
+
 ### main
 
 GetOptions (
@@ -378,6 +393,12 @@ for my $url (@ARGV) {
     } else {
       die "unsupported repo type: $type\n";
     }
+  } elsif ($url =~ /^http:\/\/([^\/]*)\/?/) {
+    my $repoid = md5_hex($url);
+    my $cachedir = ($::ENV{'CACHE_DIR'} || '/var/cache/build');
+    $dir = "$cachedir/$repoid/";
+    getmetadata($url, $dir);
+    $baseurl = $url;
   } else {
     $dir = $url;
     $dir .= '/' unless $dir =~ /\/$/;
@@ -395,6 +416,9 @@ for my $url (@ARGV) {
     @cursor = ([undef, $primaryparser]);
 
     my $u = $dir . $f->{'location'};
+    if ($url =~ /^http:\/\/([^\/]*)\/?/) {
+        system($INC[0].'/download', $dir . "repodata/", $baseurl . "repodata/" . basename($u));
+    }
     $u = 'gzip -cd ' . $u . '|' if ($u =~ /\.gz$/); # XXX
 
     my $fh;

diff --git a/init_buildsystem b/init_buildsystem
index 6d74d2c..6abde32 100755 (executable)
--- a/init_buildsystem
+++ b/init_buildsystem
@@ -76,6 +76,10 @@ while test -n "$1" ; do
            CLEAN_BUILD="$1"
            shift
        ;;
+       --cachedir)
+           CACHE_DIR="$2"
+           shift 2
+           ;;
        *)
            break
            ;;
@@ -285,6 +289,9 @@ function validate_cache_file {
            test -z "$SRC" && SRC=`pwd`
            if [ "${SRC#zypp://}" != "$SRC" ]; then
                set -- $BUILD_DIR/createrepomddeps "$SRC"
+           elif [ "${SRC#http://}" != "$SRC" ]; then
+               mkdir -p "$(getcachedir "$SRC")"
+               set -- env CACHE_DIR="$CACHE_DIR" $BUILD_DIR/createrepomddeps "$SRC"
            elif [ ! -e "$SRC" ]; then
                echo "*** $SRC does not exist" >&2
                cleanup_and_exit 1
@@ -339,13 +346,26 @@ fail_exit()
 }
 
 # modify $SRC
-download_zypp()
+downloadpkg()
 {
     local url="$1"
-    local zd="/var/cache/zypp/packages/"
-    SRC=$zd${url#zypp://}
+    local cachedir
+
+    if [ "${url:0:7}" == "zypp://" ] ; then
+       cachedir="/var/cache/zypp/packages/"
+       SRC="$cachedir${url#zypp://}"
+       mkdir -p "$(dirname $SRC)"
+    elif [ "${url:0:7}" == "http://" ] ; then
+       cachedir="$(getcachedir "$url")"
+       local name="$(basename "$url")"
+       SRC="$cachedir/$name"
+    else
+       echo "Invalid url: $url"
+       cleanup_and_exit 1
+    fi
+
     if [ ! -e "$SRC" ]; then
-       local destdir="$zd/tmp"
+       local destdir="$cachedir/tmp"
        mkdir -p "$destdir"
        echo "downloading $url ... ";
        $BUILD_DIR/download "$destdir" "$url" || cleanup_and_exit 1
@@ -364,6 +384,27 @@ download_zypp()
     fi
 }
 
+getcachedir()
+{
+    url=$1
+    for repo in "${repos[@]}" ; do
+       if [ "${url:0:${#repo}}" == "$repo" ] ; then
+           read repoid dummy < <(echo -n "$repo" | md5sum)
+           echo "$CACHE_DIR/$repoid"
+           break
+       fi
+    done
+}
+
+get_pkg_filename()
+{
+    local url="$1"
+    local name=$(basename $url)
+    local cachedir=$(getcachedir $url)
+    local destfile="$cachedir/$name"
+    echo $destfile
+}
+
 set_build_arch
 
 trap fail_exit EXIT
@@ -508,8 +549,8 @@ else
        if [ "${SRC#/}" = "$SRC" ]; then
            url="$SRC"
            case "$url" in
-               zypp://*)
-                   download_zypp "$url"
+               zypp://* | http://*)
+                   downloadpkg "$url"
                    ;;
                *)
                    echo "unsupported url: $url" >&2
@@ -517,7 +558,12 @@ else
                    ;;
            esac
        fi
-       ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}"
+       if [ "${SRC:0:7}" == "http://" ] ; then
+           target_file=$(get_pkg_filename "$SRC")
+           ln -s "$target_file" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}"
+       else
+           ln -s "$SRC" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.${SRC##*.}"
+       fi
        PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL $PKG"
     done < $RPMLIST