buildhistory: improve performance of image info collection
authorPaul Eggleton <paul.eggleton@linux.intel.com>
Mon, 23 Jul 2012 06:59:11 +0000 (07:59 +0100)
committerRichard Purdie <richard.purdie@linuxfoundation.org>
Sun, 29 Jul 2012 09:16:15 +0000 (10:16 +0100)
Reduce the number of calls to the packaging tool, especially in the case
of rpm, using helper utilities to gather the required information more
efficiently where possible.

(From OE-Core rev: d0b8a98c5b46c305afd389fc862b3bf0c6f1eaab)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
meta/classes/buildhistory.bbclass
meta/classes/rootfs_deb.bbclass
meta/classes/rootfs_ipk.bbclass
meta/classes/rootfs_rpm.bbclass
meta/recipes-devtools/rpm/rpmresolve/rpmresolve.c

index f0bf849..ddb76e8 100644 (file)
@@ -285,48 +285,43 @@ buildhistory_get_image_installed() {
        mkdir -p ${BUILDHISTORY_DIR_IMAGE}
 
        # Get list of installed packages
-       list_installed_packages | sort > ${BUILDHISTORY_DIR_IMAGE}/installed-package-names.txt
-       INSTALLED_PKGS=`cat ${BUILDHISTORY_DIR_IMAGE}/installed-package-names.txt`
+       pkgcache="${BUILDHISTORY_DIR_IMAGE}/installed-packages.tmp"
+       list_installed_packages file | sort > $pkgcache
+
+       cat $pkgcache | awk '{ print $1 }' > ${BUILDHISTORY_DIR_IMAGE}/installed-package-names.txt
+       cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > ${BUILDHISTORY_DIR_IMAGE}/installed-packages.txt
+
+       # Produce dependency graph
+       # First, filter out characters that cause issues for dot
+       rootfs_list_installed_depends | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' > ${BUILDHISTORY_DIR_IMAGE}/depends.tmp
+       # Change delimiter from pipe to -> and set style for recommend lines
+       sed -i -e 's:|: -> :' -e 's:\[REC\]:[style=dotted]:' -e 's:$:;:' ${BUILDHISTORY_DIR_IMAGE}/depends.tmp
+       # Add header, sorted and de-duped contents and footer and then delete the temp file
+       echo -e "digraph depends {\n    node [shape=plaintext]" > ${BUILDHISTORY_DIR_IMAGE}/depends.dot
+       cat ${BUILDHISTORY_DIR_IMAGE}/depends.tmp | sort | uniq >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot
+       echo "}" >>  ${BUILDHISTORY_DIR_IMAGE}/depends.dot
+       rm ${BUILDHISTORY_DIR_IMAGE}/depends.tmp
 
-       # Produce installed package file and size lists and dependency graph
-       echo -n > ${BUILDHISTORY_DIR_IMAGE}/installed-packages.txt
+       # Produce installed package sizes list
        echo -n > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
-       echo -e "digraph depends {\n    node [shape=plaintext]" > ${BUILDHISTORY_DIR_IMAGE}/depends.dot
-       for pkg in $INSTALLED_PKGS; do
-               pkgfile=`get_package_filename $pkg`
-               echo `basename $pkgfile` >> ${BUILDHISTORY_DIR_IMAGE}/installed-packages.txt
+       cat $pkgcache | while read pkg pkgfile
+       do
                if [ -f $pkgfile ] ; then
                        pkgsize=`du -k $pkgfile | head -n1 | awk '{ print $1 }'`
                        echo $pkgsize $pkg >> ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
                fi
-
-               deps=`list_package_depends $pkg`
-               for dep in $deps ; do
-                       echo "$pkg OPP $dep;" | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' | sed 's:OPP:->:g'
-               done
-
-               recs=`list_package_recommends $pkg`
-               for rec in $recs ; do
-                       echo "$pkg OPP $rec [style=dotted];" | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' | sed 's:OPP:->:g'
-               done
-       done | sort | uniq >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot
-       echo "}" >>  ${BUILDHISTORY_DIR_IMAGE}/depends.dot
-
+       done
        cat ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.txt
        rm ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
 
+       # We're now done with the cache, delete it
+       rm $pkgcache
+
        # Produce some cut-down graphs (for readability)
        grep -v kernel_image ${BUILDHISTORY_DIR_IMAGE}/depends.dot | grep -v kernel_2 | grep -v kernel_3 > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel.dot
        grep -v libc6 ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel.dot | grep -v libgcc > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc.dot
        grep -v update_ ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc.dot > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate.dot
        grep -v kernel_module ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate.dot > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate-nomodules.dot
-
-       # Workaround for broken shell function dependencies
-       if false ; then
-               get_package_filename
-               list_package_depends
-               list_package_recommends
-       fi
 }
 
 buildhistory_get_imageinfo() {
index a002b1e..750a8ca 100644 (file)
@@ -96,26 +96,24 @@ list_installed_packages() {
        if [ "$1" = "arch" ] ; then
                # Here we want the PACKAGE_ARCH not the deb architecture
                ${DPKG_QUERY_COMMAND} -W -f='${Package} ${PackageArch}\n'
+       elif [ "$1" = "file" ] ; then
+               ${DPKG_QUERY_COMMAND} -W -f='${Package} ${Package}_${Version}_${Architecture}.deb\n' | while read pkg pkgfile
+               do
+                       fullpath=`find ${DEPLOY_DIR_DEB} -name "$pkgfile" || true`
+                       if [ "$fullpath" = "" ] ; then
+                               echo "$pkg $pkgfile"
+                       else
+                               echo "$pkg $fullpath"
+                       fi
+               done
        else
                ${DPKG_QUERY_COMMAND} -W -f='${Package}\n'
        fi
 }
 
-get_package_filename() {
-       fullname=`find ${DEPLOY_DIR_DEB} -name "$1_*.deb" || true`
-       if [ "$fullname" = "" ] ; then
-               echo $name
-       else
-               echo $fullname
-       fi
-}
-
-list_package_depends() {
-       ${DPKG_QUERY_COMMAND} -s $1 | grep ^Depends | sed -e 's/^Depends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
-}
-
-list_package_recommends() {
-       ${DPKG_QUERY_COMMAND} -s $1 | grep ^Recommends | sed -e 's/^Recommends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
+rootfs_list_installed_depends() {
+       # Cheat here a little bit by using the opkg query helper util
+       ${DPKG_QUERY_COMMAND} -W -f='Package: ${Package}\nDepends: ${Depends}\nRecommends: ${Recommends}\n\n' | opkg-query-helper.py
 }
 
 rootfs_install_packages() {
index 7df97a0..6cdd8f6 100644 (file)
@@ -131,33 +131,23 @@ remove_packaging_data_files() {
 list_installed_packages() {
        if [ "$1" = "arch" ] ; then
                opkg-cl ${IPKG_ARGS_POST} status | opkg-query-helper.py -a
+       elif [ "$1" = "file" ] ; then
+               opkg-cl ${IPKG_ARGS_POST} status | opkg-query-helper.py -f | while read pkg pkgfile
+               do
+                       fullpath=`find ${DEPLOY_DIR_IPK} -name "$pkgfile" || true`
+                       if [ "$fullpath" = "" ] ; then
+                               echo "$pkg $pkgfile"
+                       else
+                               echo "$pkg $fullpath"
+                       fi
+               done
        else
                opkg-cl ${IPKG_ARGS_POST} list_installed | awk '{ print $1 }'
        fi
 }
 
-get_package_filename() {
-       set +x
-       info=`opkg-cl ${IPKG_ARGS_POST} info $1 | grep -B 7 -A 7 "^Status.* \(\(installed\)\|\(unpacked\)\)" || true`
-       name=`echo "${info}" | awk '/^Package/ {printf $2"_"}'`
-       name=$name`echo "${info}" | awk -F: '/^Version/ {printf $NF"_"}' | sed 's/^\s*//g'`
-       name=$name`echo "${info}" | awk '/^Archi/ {print $2".ipk"}'`
-       set -x
-
-       fullname=`find ${DEPLOY_DIR_IPK} -name "$name" || true`
-       if [ "$fullname" = "" ] ; then
-               echo $name
-       else
-               echo $fullname
-       fi
-}
-
-list_package_depends() {
-       opkg-cl ${IPKG_ARGS_POST} info $1 | grep ^Depends | sed -e 's/^Depends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
-}
-
-list_package_recommends() {
-       opkg-cl ${IPKG_ARGS_POST} info $1 | grep ^Recommends | sed -e 's/^Recommends: //' -e 's/,//g' -e 's:([=<>]* [^ )]*)::g'
+rootfs_list_installed_depends() {
+       opkg-cl ${IPKG_ARGS_POST} status | opkg-query-helper.py
 }
 
 rootfs_install_packages() {
index 1cc4a84..c9258df 100644 (file)
@@ -143,40 +143,15 @@ RPM_QUERY_CMD = '${RPM} --root $INSTALL_ROOTFS_RPM -D "_dbpath ${rpmlibdir}" \
 list_installed_packages() {
        if [ "$1" = "arch" ] ; then
                ${RPM_QUERY_CMD} -qa --qf "[%{NAME} %{ARCH}\n]"
+       elif [ "$1" = "file" ] ; then
+               ${RPM_QUERY_CMD} -qa --qf "[%{NAME} %{PACKAGEORIGIN}\n]"
        else
                ${RPM_QUERY_CMD} -qa --qf "[%{NAME}\n]"
        fi
 }
 
-get_package_filename() {
-       resolve_package_rpm ${RPMCONF_TARGET_BASE}-base_archs.conf $1
-}
-
-list_package_depends() {
-       pkglist=`list_installed_packages`
-
-       # REQUIRE* lists "soft" requirements (which we know as recommends and RPM refers to
-       # as "suggests") so filter these out with the help of awk
-       for req in `${RPM_QUERY_CMD} -q --qf "[%{REQUIRENAME} %{REQUIREFLAGS}\n]" $1 | awk '{ if( and($2, 0x80000) == 0) print $1 }'`; do
-               if echo "$req" | grep -q "^rpmlib" ; then continue ; fi
-
-               realpkg=""
-               for dep in $pkglist; do
-                       if [ "$dep" = "$req" ] ; then
-                               realpkg="1"
-                               echo $req
-                               break
-                       fi
-               done
-
-               if [ "$realdep" = "" ] ; then
-                       ${RPM_QUERY_CMD} -q --whatprovides $req --qf "%{NAME}\n"
-               fi
-       done
-}
-
-list_package_recommends() {
-       ${RPM_QUERY_CMD} -q --suggests $1
+rootfs_list_installed_depends() {
+       rpmresolve -d $INSTALL_ROOTFS_RPM/${rpmlibdir}
 }
 
 rootfs_install_packages() {
index 9f6cdf2..2d9ed14 100644 (file)
@@ -204,10 +204,104 @@ int processPackages(rpmts *ts, int tscount, const char *packagelistfn, int ignor
     return rc;
 }
 
+int lookupProvider(rpmts ts, const char *req, char **provider)
+{
+    int rc = 0;
+    rpmmi provmi = rpmtsInitIterator(ts, RPMTAG_PROVIDENAME, req, 0);
+    if(provmi) {
+        Header h;
+        if ((h = rpmmiNext(provmi)) != NULL) {
+            HE_t he = (HE_t) memset(alloca(sizeof(*he)), 0, sizeof(*he));
+            he->tag = RPMTAG_NAME;
+            rc = (headerGet(h, he, 0) != 1);
+            if(rc==0)
+                *provider = strdup((char *)he->p.ptr);
+        }
+        (void)rpmmiFree(provmi);
+    }
+    else {
+        rc = -1;
+    }
+    return rc;
+}
+
+int printDepList(rpmts *ts, int tscount)
+{
+    int rc = 0;
+
+    if( tscount > 1 )
+        printf(">1 database specified with dependency list, using first only\n");
+
+    /* Get list of names */
+    rpmdb db = rpmtsGetRdb(ts[0]);
+    ARGV_t names = NULL;
+    rc = rpmdbMireApply(db, RPMTAG_NAME,
+                RPMMIRE_STRCMP, NULL, &names);
+    int nnames = argvCount(names);
+
+    /* Get list of NVRAs */
+    ARGV_t keys = NULL;
+    rc = rpmdbMireApply(db, RPMTAG_NVRA,
+                RPMMIRE_STRCMP, NULL, &keys);
+    if (keys) {
+        int i, j;
+        HE_t he = (HE_t) memset(alloca(sizeof(*he)), 0, sizeof(*he));
+        int nkeys = argvCount(keys);
+        for(i=0; i<nkeys; i++) {
+            rpmmi mi = rpmtsInitIterator(ts[0], RPMTAG_NVRA, keys[i], 0);
+            Header h;
+            if ((h = rpmmiNext(mi)) != NULL) {
+                /* Get name of package */
+                he->tag = RPMTAG_NAME;
+                rc = (headerGet(h, he, 0) != 1);
+                char *name = strdup((char *)he->p.ptr);
+                /* Get its requires */
+                he->tag = RPMTAG_REQUIRENAME;
+                rc = (headerGet(h, he, 0) != 1);
+                ARGV_t reqs = (ARGV_t)he->p.ptr;
+                /* Get its requireflags */
+                he->tag = RPMTAG_REQUIREFLAGS;
+                rc = (headerGet(h, he, 0) != 1);
+                rpmuint32_t *reqflags = (rpmuint32_t *)he->p.ui32p;
+                for(j=0; j<he->c; j++) {
+                    int k;
+                    char *prov = NULL;
+                    for(k=0; k<nnames; k++) {
+                        if(strcmp(names[k], reqs[j]) == 0) {
+                            prov = names[k];
+                            break;
+                        }
+                    }
+                    if(prov) {
+                        if((int)reqflags[j] & 0x80000)
+                            printf("%s|%s [REC]\n", name, prov);
+                        else
+                            printf("%s|%s\n", name, prov);
+                    }
+                    else {
+                        rc = lookupProvider(ts[0], reqs[j], &prov);
+                        if(rc==0 && prov) {
+                            if((int)reqflags[j] & 0x80000)
+                                printf("%s|%s [REC]\n", name, prov);
+                            else
+                                printf("%s|%s\n", name, prov);
+                            free(prov);
+                        }
+                    }
+                }
+                free(name);
+            }
+            (void)rpmmiFree(mi);
+        }
+    }
+
+    return rc;
+}
+
 void usage()
 {
     fprintf(stderr, "OpenEmbedded rpm resolver utility\n");
-    fprintf(stderr, "syntax: rpmresolve [-i] <dblistfile> <packagelistfile>\n");
+    fprintf(stderr, "syntax: rpmresolve [-i] [-d] <dblistfile> <packagelistfile>\n");
 }
 
 int main(int argc, char **argv)
@@ -218,13 +312,17 @@ int main(int argc, char **argv)
     int i;
     int c;
     int ignoremissing = 0;
+    int deplistmode = 0;
 
     opterr = 0;
-    while ((c = getopt (argc, argv, "i")) != -1) {
+    while ((c = getopt (argc, argv, "id")) != -1) {
         switch (c) {
             case 'i':
                 ignoremissing = 1;
                 break;
+            case 'd':
+                deplistmode = 1;
+                break;
             case '?':
                 if(isprint(optopt))
                     fprintf(stderr, "Unknown option `-%c'.\n", optopt);
@@ -258,12 +356,17 @@ int main(int argc, char **argv)
         return 1;
     }
 
-    if( argc - optind < 2 ) {
-        fprintf(stderr, "Please specify package list file\n");
-        return 1;
+    if(deplistmode) {
+        rc = printDepList(ts, tscount);
+    }
+    else {
+        if( argc - optind < 2 ) {
+            fprintf(stderr, "Please specify package list file\n");
+            return 1;
+        }
+        const char *pkglistfn = argv[optind+1];
+        rc = processPackages(ts, tscount, pkglistfn, ignoremissing);
     }
-    const char *pkglistfn = argv[optind+1];
-    rc = processPackages(ts, tscount, pkglistfn, ignoremissing);
 
     for(i=0; i<tscount; i++)
         (void) rpmtsCloseDB(ts[i]);