Patch Coverage tool 68/79768/3
authorDavid Steele <david.steele@samsung.com>
Tue, 12 Jul 2016 16:51:21 +0000 (17:51 +0100)
committerDavid Steele <david.steele@samsung.com>
Fri, 15 Jul 2016 14:08:36 +0000 (07:08 -0700)
Change-Id: I238054661978a27b15704e38bda496b7fabc699d

automated-tests/patch-coverage.pl [new file with mode: 0755]
build/tizen/Makefile.am

diff --git a/automated-tests/patch-coverage.pl b/automated-tests/patch-coverage.pl
new file mode 100755 (executable)
index 0000000..68754eb
--- /dev/null
@@ -0,0 +1,555 @@
+#!/usr/bin/perl
+#
+# Copyright (c) 2016 Samsung Electronics Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use strict;
+use Git;
+use Getopt::Long;
+use Error qw(:try);
+use Pod::Usage;
+use File::Basename;
+use File::stat;
+use Scalar::Util qw /looks_like_number/;
+use Cwd qw /getcwd/;
+use Term::ANSIColor qw(:constants);
+
+# Program to run gcov on files in patch (that are in source dirs - needs to be dali-aware).
+
+# A) Get patch
+# B) Remove uninteresting files
+# C) Find matching gcno/gcda files
+# D) Copy and rename them to match source prefix (i.e. strip library name off front)
+# E) Generate patch output with covered/uncovered lines marked in green/red
+# F) Generate coverage data for changed lines
+# G) Exit status should be 0 for high coverage (90% line coverage for all new/changed lines)
+#    or 1 for low coverage
+
+# Sources for conversion of gcno/gcda files:
+# ~/bin/lcov
+# Python git-coverage (From http://stef.thewalter.net/git-coverage-useful-code-coverage.html)
+
+our $repo = Git->repository();
+our $debug=0;
+our $pd_debug=0;
+our $opt_cached;
+our $opt_head;
+#our $opt_workingtree;
+#our $opt_diff=1;
+our $opt_help;
+our $opt_verbose;
+our $opt_quiet;
+
+my %options = (
+    "cached"       => { "optvar"=>\$opt_cached, "desc"=>"Use index" },
+    "head"         => { "optvar"=>\$opt_head, "desc"=>"Use git show" },
+    "help"         => { "optvar"=>\$opt_help, "desc"=>""},
+    "quiet"        => { "optvar"=>\$opt_quiet, "desc"=>""},
+    "verbose"      => { "optvar"=>\$opt_verbose, "desc"=>"" });
+
+my %longOptions = map { $_ => $options{$_}->{"optvar"} } keys(%options);
+GetOptions( %longOptions ) or pod2usage(2);
+pod2usage(1) if $opt_help;
+
+
+## Format per file, repeated, no linebreak
+# <diffcmd>
+# index c1..c2 c3
+# --- a/<left-hand-side-file>
+# +++ b/<right-hand-side-file>
+# <diff hunks>
+
+# Format of each diff hunk, repeated, no linebreak
+# @@ <ranges> @@ line
+# 3 lines of context
+# [-|+]lines removed on left, added on right
+# 3 lines of context
+#
+# output:
+sub parse_diff
+{
+    my $patchref = shift;
+    my $file="";
+    my @checklines=();
+    my %b_lines=();
+    my $state = 0;
+    my $store_line=-1;
+    my %files=();
+
+    print "Patch size: ".scalar(@$patchref)."\n" if $pd_debug;
+    for my $line (@$patchref)
+    {
+        if($state == 0)
+        {
+            print "State: $state  $line  \n" if $pd_debug;
+            # Search for a line matching "+++ b/<filename>"
+            if( $line =~ m!^\+\+\+ b/([\w-_\./]*)!)
+            {
+                $file = $1;
+                $state = 1 ;
+                print "Found File: $file\n" if $pd_debug;
+            }
+        }
+        else #elsif($state == 1)
+        {
+            # If we find a line starting with diff, the previous
+            # file's diffs have finished, store them.
+            if( $line =~ /^diff/)
+            {
+                print "State: $state  $line  \n" if $pd_debug;
+                $state = 0;
+                # if the file had changes, store the new/modified line numbers
+                if( $file && scalar(@checklines))
+                {
+                    $files{$file}->{"patch"} = [@checklines];
+                    $files{$file}->{"b_lines"} = {%b_lines};
+                    @checklines=();
+                    %b_lines=();
+                }
+                print("\n\n") if $pd_debug;
+            }
+            # If we find a line starting with @@, it tells us the line numbers
+            # of the old file and new file for this hunk.
+            elsif( $line =~ /^@@/)
+            {
+                print "State: $state  $line  \n" if $pd_debug;
+
+                # Find the lines in the new file (of the form "+<start>[,<length>])
+                my($start,$space,$length) = ($line =~ /\+([0-9]+)(,| )([0-9]+)?/);
+                if($length || $space eq " ")
+                {
+                    if( $space eq " " )
+                    {
+                        $length=1;
+                    }
+                    push(@checklines, [$start, $length]);
+                    $store_line=$start;
+                }
+                else
+                {
+                    $store_line = -1;
+                }
+                if($pd_debug)
+                {
+                    my $last = scalar(@checklines)-1;
+                    if( $last >= 0 )
+                    {
+                        print "Checkline:" . $checklines[$last]->[0] . ", " . $checklines[$last]->[1] . "\n";
+                    }
+                }
+            }
+            # If we find a line starting with "+", it belongs to the new file's patch
+            elsif( $line =~ /^\+/)
+            {
+               if($store_line >= 0)
+               {
+                   chomp;
+                   $line = substr($line, 1); # Remove leading +
+                   $b_lines{$store_line} = $line;
+                   $store_line++;
+               }
+            }
+        }
+    }
+    # Store the final entry
+    $files{$file}->{"patch"} = [@checklines];
+    $files{$file}->{"b_lines"} = {%b_lines};
+
+    my %filter = map { $_ => $files{$_} } grep {m!^dali(-toolkit)?/!} (keys(%files));;
+
+    if($pd_debug)
+    {
+        print("Filtered files:\n");
+        foreach my $file (keys(%filter))
+        {
+            print("$file: ");
+            $patchref = $filter{$file}->{"patch"};
+            foreach my $lineblock (@$patchref)
+            {
+                print "$lineblock->[0]($lineblock->[1]) "
+            }
+            print ( "\n");
+        }
+    }
+
+    return {%filter};
+}
+
+sub show_patch_lines
+{
+    my $filesref = shift;
+    print "\nNumber of files: " . scalar(keys(%$filesref)) . "\n";
+    for my $file (keys(%$filesref))
+    {
+        print("$file:");
+        my $clref = $filesref->{$file}->{"patch"};
+        for my $cl (@$clref)
+        {
+            print("($cl->[0],$cl->[1]) ");
+        }
+        print("\n");
+    }
+}
+
+sub get_gcno_file
+{
+    # Assumes test cases have been run, and "make rename_cov_data" has been executed
+
+    my $file = shift;
+    my ($name, $path, $suffix) = fileparse($file, (".c", ".cpp", ".h"));
+    my $gcno_file = $repo->wc_path() . "/build/tizen/.cov/$name.gcno";
+
+    # Note, will translate headers to their source's object, which
+    # may miss execution code in the headers (e.g. inlines are usually
+    # not all used in the implementation, and require getting coverage
+    # from test cases.
+
+    if( -f $gcno_file )
+    {
+        my $gcno_st = stat($gcno_file);
+        my $fq_file = $repo->wc_path() . $file;
+        my $src_st = stat($fq_file);
+        if($gcno_st->ctime < $src_st->mtime)
+        {
+            print "WARNING: GCNO $gcno_file older than SRC $fq_file\n";
+            $gcno_file="";
+        }
+
+    }
+    else
+    {
+        print("WARNING: No equivalent gcno file for $file\n");
+    }
+    return $gcno_file;
+}
+
+our %gcovfiles=();
+sub get_coverage
+{
+    my $file = shift;
+    my $filesref = shift;
+    print("get_coverage($file)\n") if $debug;
+
+    my $gcno_file = get_gcno_file($file);
+    my @gcov_files = ();
+    my $gcovfile;
+    if( $gcno_file )
+    {
+        print "Running gcov on $gcno_file:\n" if $debug;
+        open( my $fh,  "gcov --preserve-paths $gcno_file |") || die "Can't run gcov:$!\n";
+        while( <$fh> )
+        {
+            print $_ if $debug>=3;
+            chomp;
+            if( m!'(.*\.gcov)'$! )
+            {
+                my $coverage_file = $1; # File has / replaced with # and .. replaced with ^
+                my $source_file = $coverage_file;
+                $source_file =~ s!\^!..!g;  # Change ^ to ..
+                $source_file =~ s!\#!/!g;   # change #'s to /s
+                $source_file =~ s!.gcov$!!; # Strip off .gcov suffix
+
+                print "Matching $file against $source_file\n" if $debug >= 3;
+                # Only want the coverage files matching source file:
+                if(index( $source_file, $file ) > 0 )
+                {
+                    $gcovfile = $coverage_file;
+                    last;
+                }
+            }
+        }
+        close($fh);
+
+        if($gcovfile)
+        {
+            if($gcovfiles{$gcovfile} == undef)
+            {
+                # Only parse a gcov file once
+                $gcovfiles{$gcovfile}->{"seen"}=1;
+
+                print "Getting coverage data from $gcovfile\n" if $debug;
+
+                open( FH, "< $gcovfile" ) || die "Can't open $gcovfile for reading:$!\n";
+                while(<FH>)
+                {
+                    my ($cov, $line, @code ) = split( /:/, $_ );
+                    $cov =~ s/^\s+//; # Strip leading space
+                    $line =~ s/^\s+//;
+                    my $code=join(":", @code);
+                    if($cov =~ /\#/)
+                    {
+                        # There is no coverage data for these executable lines
+                        $gcovfiles{$gcovfile}->{"uncovered"}->{$line}++;
+                        $gcovfiles{$gcovfile}->{"src"}->{$line}=$code;
+                    }
+                    elsif( $cov ne "-" && looks_like_number($cov) && $cov > 0 )
+                    {
+                        $gcovfiles{$gcovfile}->{"covered"}->{$line}=$cov;
+                        $gcovfiles{$gcovfile}->{"src"}->{$line}=$code;
+                    }
+                    else
+                    {
+                        # All other lines are not executable.
+                        $gcovfiles{$gcovfile}->{"src"}->{$line}=$code;
+                    }
+                }
+                close( FH );
+            }
+            $filesref->{$file}->{"coverage"} = $gcovfiles{$gcovfile}; # store hashref
+        }
+        else
+        {
+            # No gcov output - the gcno file produced no coverage of the source/header
+            # Probably means that there is no coverage for the file (with the given
+            # test case - there may be some somewhere, but for the sake of speed, don't
+            # check (yet).
+        }
+    }
+}
+
+# Run the git diff command to get the patch, then check the coverage
+# output for the patch.
+sub run_diff
+{
+    my ($fh, $c) = $repo->command_output_pipe(@_);
+    our @patch=();
+    while(<$fh>)
+    {
+        chomp;
+        push @patch, $_;
+    }
+    $repo->command_close_pipe($fh, $c);
+
+    # @patch has slurped diff for all files...
+    my $filesref = parse_diff ( \@patch );
+    show_patch_lines($filesref) if $debug;
+
+    print "Checking coverage:\n" if $debug;
+
+    my $cwd=getcwd();
+    chdir ".cov" || die "Can't find $cwd/.cov:$!\n";
+
+    for my $file (keys(%$filesref))
+    {
+        my ($name, $path, $suffix) = fileparse($file, qr{\.[^.]*$});
+        if($suffix eq ".cpp" || $suffix eq ".c" || $suffix eq ".h")
+        {
+            get_coverage($file, $filesref);
+        }
+    }
+    chdir $cwd;
+    return $filesref;
+}
+
+
+sub calc_patch_coverage_percentage
+{
+    my $filesref = shift;
+    my $total_covered_lines = 0;
+    my $total_uncovered_lines = 0;
+
+    foreach my $file (keys(%$filesref))
+    {
+        my $covered_lines = 0;
+        my $uncovered_lines = 0;
+
+        my $patchref = $filesref->{$file}->{"patch"};
+        my $coverage_ref = $filesref->{$file}->{"coverage"};
+        if( $coverage_ref )
+        {
+            for my $patch (@$patchref)
+            {
+                for(my $i = 0; $i < $patch->[1]; $i++ )
+                {
+                    my $line = $i + $patch->[0];
+                    if($coverage_ref->{"covered"}->{$line})
+                    {
+                        $covered_lines++;
+                        $total_covered_lines++;
+                    }
+                    if($coverage_ref->{"uncovered"}->{$line})
+                    {
+                        $uncovered_lines++;
+                        $total_uncovered_lines++;
+                    }
+                }
+            }
+            $coverage_ref->{"covered_lines"} = $covered_lines;
+            $coverage_ref->{"uncovered_lines"} = $uncovered_lines;
+            my $total = $covered_lines + $uncovered_lines;
+            my $percent = 0;
+            if($total > 0)
+            {
+                $percent = $covered_lines / $total;
+            }
+            $coverage_ref->{"percent_covered"} = 100 * $percent;
+        }
+    }
+    my $total_exec = $total_covered_lines + $total_uncovered_lines;
+    my $percent = 0;
+    if($total_exec > 0) { $percent = 100 * $total_covered_lines / $total_exec; }
+
+    return $percent;
+}
+
+sub patch_output
+{
+    my $filesref = shift;
+    foreach my $file (keys(%$filesref))
+    {
+        my ($name, $path, $suffix) = fileparse($file, qr{\.[^.]*$});
+        next if($path !~ /^dali/);
+
+        my $patchref = $filesref->{$file}->{"patch"};
+        my $b_lines_ref = $filesref->{$file}->{"b_lines"};
+        my $coverage_ref = $filesref->{$file}->{"coverage"};
+        print BOLD, "$file  ";
+
+        if($coverage_ref)
+        {
+            if( $coverage_ref->{"covered_lines"} > 0
+                ||
+                $coverage_ref->{"uncovered_lines"} > 0 )
+            {
+                print GREEN, "Covered: " . $coverage_ref->{"covered_lines"}, RED, " Uncovered: " . $coverage_ref->{"uncovered_lines"}, RESET;
+            }
+        }
+        else
+        {
+            if($suffix eq ".cpp" || $suffix eq ".c" || $suffix eq ".h")
+            {
+                print RED;
+            }
+            print "No coverage found";
+        }
+        print RESET "\n";
+
+        for my $patch (@$patchref)
+        {
+            my $hunkstr="Hunk: " . $patch->[0];
+            if( $patch->[1] > 1 )
+            {
+                $hunkstr .= " - " . ($patch->[0]+$patch->[1]-1);
+            }
+            print BOLD, "$hunkstr\n",  RESET;
+            for(my $i = 0; $i < $patch->[1]; $i++ )
+            {
+                my $line = $i + $patch->[0];
+                printf "%-6s  ", $line;
+
+                if($coverage_ref)
+                {
+                    my $color;
+                    if($coverage_ref->{"covered"}->{$line})
+                    {
+                        $color=GREEN;
+                    }
+                    elsif($coverage_ref->{"uncovered"}->{$line})
+                    {
+                        $color=BOLD . RED;
+                    }
+                    else
+                    {
+                        $color=BLACK;
+                    }
+                    my $src=$coverage_ref->{"src"}->{$line};
+                    chomp($src);
+                    print $color, "$src\n", RESET;
+                }
+                else
+                {
+                    # We don't have coverage data, so print it from the patch instead.
+                    my $src = $b_lines_ref->{$line};
+                    print "$src\n";
+                }
+            }
+        }
+    }
+}
+
+
+################################################################################
+##                                    MAIN                                    ##
+################################################################################
+
+my $cwd = getcwd();
+chdir $repo->wc_path();
+chdir "build/tizen";
+`make rename_cov_data`;
+
+my @cmd=('--no-pager','diff','--no-ext-diff','-U0','--no-color');
+
+my $status = $repo->command("status", "-s");
+if( $status eq "" )
+{
+    # There are no changes in the index or working tree. Use the last patch instead
+    push @cmd, ('HEAD~1','HEAD');
+}
+elsif($opt_cached) # TODO: Remove this option. Instead, need full diff
+{
+    push @cmd, "--cached";
+}
+
+push @cmd, @ARGV;
+my $filesref = run_diff(@cmd);
+
+my $percent = calc_patch_coverage_percentage($filesref);
+if( ! $opt_quiet )
+{
+    patch_output($filesref);
+    my $color=BOLD RED;
+    if($percent>=90)
+    {
+        $color=GREEN;
+    }
+    printf("Percentage of change covered: $color %5.2f%\n" . RESET, $percent);
+}
+exit($percent<90);
+
+
+__END__
+
+=head1 NAME
+
+patch-coverage
+
+=head1 SYNOPSIS
+
+patch-coverage.pl - Determine if patch coverage is below 90%
+
+=head1 DESCRIPTION
+Calculates how well the most recent patch is covered (either the
+patch that is in the index+working directory, or HEAD).
+
+=head1 OPTIONS
+
+=over 28
+
+=item B<-c|--cached>
+Use index files if there is nothing in the working tree
+
+=item B<   --help>
+This help
+
+=item B<-q|--quiet>
+Don't generate any output
+
+=head1 RETURN STATUS
+0 if the coverage of source files is > 90%, otherwise 1
+
+=head1 EXAMPLES
+
+
+=cut
index 925f052..7d9bf95 100644 (file)
@@ -40,7 +40,7 @@ COVERAGE_OUTPUT_DIR=doc/coverage
 # From lcov version 1.10 onwards, branch coverage is off by default and earlier versions do not support the rc option
 LCOV_OPTS=`if [ \`printf "\\\`lcov --version | cut -d' ' -f4\\\`\n1.10\n" | sort -V | head -n 1\` = 1.10 ] ; then echo "--rc lcov_branch_coverage=1" ; fi`
 
-cov_data:
+rename_cov_data:
        @test -z $(COVERAGE_DIR) || mkdir -p $(COVERAGE_DIR)
        @rm -f $(COVERAGE_DIR)/*
        @cp dali-core/.libs/*.gcda dali-core/.libs/*.gcno $(COVERAGE_DIR)
@@ -48,6 +48,8 @@ cov_data:
                do mv $$i `echo $$i | sed s/libdali_core_la-//` ; echo $$i ; done
        @for i in `find $(COVERAGE_DIR) -name "libdali_la-*.gcda" -o -name "libdali_la-*.gcno"` ;\
                do mv $$i `echo $$i | sed s/libdali_la-//` ; done
+
+cov_data: rename_cov_data
        @cd $(COVERAGE_DIR) ; lcov $(LCOV_OPTS) --base-directory . --directory . -c -o dali.info
        @cd $(COVERAGE_DIR) ; lcov $(LCOV_OPTS) --remove dali.info "*/dali-env/*" "/usr/include/*" "public-api/shader-effects/*" "*/image-actor*" -o dali.info