From: David Steele Date: Tue, 12 Jul 2016 16:51:21 +0000 (+0100) Subject: Patch Coverage tool X-Git-Tag: dali_1.1.44~3 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=0ef97d0335c286e1351e9aa29c56dd6fb56231a3;p=platform%2Fcore%2Fuifw%2Fdali-core.git Patch Coverage tool Change-Id: I238054661978a27b15704e38bda496b7fabc699d --- diff --git a/automated-tests/patch-coverage.pl b/automated-tests/patch-coverage.pl new file mode 100755 index 000000000..68754eb53 --- /dev/null +++ b/automated-tests/patch-coverage.pl @@ -0,0 +1,555 @@ +#!/usr/bin/perl +# +# Copyright (c) 2016 Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +use strict; +use Git; +use Getopt::Long; +use Error qw(:try); +use Pod::Usage; +use File::Basename; +use File::stat; +use Scalar::Util qw /looks_like_number/; +use Cwd qw /getcwd/; +use Term::ANSIColor qw(:constants); + +# Program to run gcov on files in patch (that are in source dirs - needs to be dali-aware). + +# A) Get patch +# B) Remove uninteresting files +# C) Find matching gcno/gcda files +# D) Copy and rename them to match source prefix (i.e. strip library name off front) +# E) Generate patch output with covered/uncovered lines marked in green/red +# F) Generate coverage data for changed lines +# G) Exit status should be 0 for high coverage (90% line coverage for all new/changed lines) +# or 1 for low coverage + +# Sources for conversion of gcno/gcda files: +# ~/bin/lcov +# Python git-coverage (From http://stef.thewalter.net/git-coverage-useful-code-coverage.html) + +our $repo = Git->repository(); +our $debug=0; +our $pd_debug=0; +our $opt_cached; +our $opt_head; +#our $opt_workingtree; +#our $opt_diff=1; +our $opt_help; +our $opt_verbose; +our $opt_quiet; + +my %options = ( + "cached" => { "optvar"=>\$opt_cached, "desc"=>"Use index" }, + "head" => { "optvar"=>\$opt_head, "desc"=>"Use git show" }, + "help" => { "optvar"=>\$opt_help, "desc"=>""}, + "quiet" => { "optvar"=>\$opt_quiet, "desc"=>""}, + "verbose" => { "optvar"=>\$opt_verbose, "desc"=>"" }); + +my %longOptions = map { $_ => $options{$_}->{"optvar"} } keys(%options); +GetOptions( %longOptions ) or pod2usage(2); +pod2usage(1) if $opt_help; + + +## Format per file, repeated, no linebreak +# +# index c1..c2 c3 +# --- a/ +# +++ b/ +# + +# Format of each diff hunk, repeated, no linebreak +# @@ @@ line +# 3 lines of context +# [-|+]lines removed on left, added on right +# 3 lines of context +# +# output: +sub parse_diff +{ + my $patchref = shift; + my $file=""; + my @checklines=(); + my %b_lines=(); + my $state = 0; + my $store_line=-1; + my %files=(); + + print "Patch size: ".scalar(@$patchref)."\n" if $pd_debug; + for my $line (@$patchref) + { + if($state == 0) + { + print "State: $state $line \n" if $pd_debug; + # Search for a line matching "+++ b/" + if( $line =~ m!^\+\+\+ b/([\w-_\./]*)!) + { + $file = $1; + $state = 1 ; + print "Found File: $file\n" if $pd_debug; + } + } + else #elsif($state == 1) + { + # If we find a line starting with diff, the previous + # file's diffs have finished, store them. + if( $line =~ /^diff/) + { + print "State: $state $line \n" if $pd_debug; + $state = 0; + # if the file had changes, store the new/modified line numbers + if( $file && scalar(@checklines)) + { + $files{$file}->{"patch"} = [@checklines]; + $files{$file}->{"b_lines"} = {%b_lines}; + @checklines=(); + %b_lines=(); + } + print("\n\n") if $pd_debug; + } + # If we find a line starting with @@, it tells us the line numbers + # of the old file and new file for this hunk. + elsif( $line =~ /^@@/) + { + print "State: $state $line \n" if $pd_debug; + + # Find the lines in the new file (of the form "+[,]) + my($start,$space,$length) = ($line =~ /\+([0-9]+)(,| )([0-9]+)?/); + if($length || $space eq " ") + { + if( $space eq " " ) + { + $length=1; + } + push(@checklines, [$start, $length]); + $store_line=$start; + } + else + { + $store_line = -1; + } + if($pd_debug) + { + my $last = scalar(@checklines)-1; + if( $last >= 0 ) + { + print "Checkline:" . $checklines[$last]->[0] . ", " . $checklines[$last]->[1] . "\n"; + } + } + } + # If we find a line starting with "+", it belongs to the new file's patch + elsif( $line =~ /^\+/) + { + if($store_line >= 0) + { + chomp; + $line = substr($line, 1); # Remove leading + + $b_lines{$store_line} = $line; + $store_line++; + } + } + } + } + # Store the final entry + $files{$file}->{"patch"} = [@checklines]; + $files{$file}->{"b_lines"} = {%b_lines}; + + my %filter = map { $_ => $files{$_} } grep {m!^dali(-toolkit)?/!} (keys(%files));; + + if($pd_debug) + { + print("Filtered files:\n"); + foreach my $file (keys(%filter)) + { + print("$file: "); + $patchref = $filter{$file}->{"patch"}; + foreach my $lineblock (@$patchref) + { + print "$lineblock->[0]($lineblock->[1]) " + } + print ( "\n"); + } + } + + return {%filter}; +} + +sub show_patch_lines +{ + my $filesref = shift; + print "\nNumber of files: " . scalar(keys(%$filesref)) . "\n"; + for my $file (keys(%$filesref)) + { + print("$file:"); + my $clref = $filesref->{$file}->{"patch"}; + for my $cl (@$clref) + { + print("($cl->[0],$cl->[1]) "); + } + print("\n"); + } +} + +sub get_gcno_file +{ + # Assumes test cases have been run, and "make rename_cov_data" has been executed + + my $file = shift; + my ($name, $path, $suffix) = fileparse($file, (".c", ".cpp", ".h")); + my $gcno_file = $repo->wc_path() . "/build/tizen/.cov/$name.gcno"; + + # Note, will translate headers to their source's object, which + # may miss execution code in the headers (e.g. inlines are usually + # not all used in the implementation, and require getting coverage + # from test cases. + + if( -f $gcno_file ) + { + my $gcno_st = stat($gcno_file); + my $fq_file = $repo->wc_path() . $file; + my $src_st = stat($fq_file); + if($gcno_st->ctime < $src_st->mtime) + { + print "WARNING: GCNO $gcno_file older than SRC $fq_file\n"; + $gcno_file=""; + } + + } + else + { + print("WARNING: No equivalent gcno file for $file\n"); + } + return $gcno_file; +} + +our %gcovfiles=(); +sub get_coverage +{ + my $file = shift; + my $filesref = shift; + print("get_coverage($file)\n") if $debug; + + my $gcno_file = get_gcno_file($file); + my @gcov_files = (); + my $gcovfile; + if( $gcno_file ) + { + print "Running gcov on $gcno_file:\n" if $debug; + open( my $fh, "gcov --preserve-paths $gcno_file |") || die "Can't run gcov:$!\n"; + while( <$fh> ) + { + print $_ if $debug>=3; + chomp; + if( m!'(.*\.gcov)'$! ) + { + my $coverage_file = $1; # File has / replaced with # and .. replaced with ^ + my $source_file = $coverage_file; + $source_file =~ s!\^!..!g; # Change ^ to .. + $source_file =~ s!\#!/!g; # change #'s to /s + $source_file =~ s!.gcov$!!; # Strip off .gcov suffix + + print "Matching $file against $source_file\n" if $debug >= 3; + # Only want the coverage files matching source file: + if(index( $source_file, $file ) > 0 ) + { + $gcovfile = $coverage_file; + last; + } + } + } + close($fh); + + if($gcovfile) + { + if($gcovfiles{$gcovfile} == undef) + { + # Only parse a gcov file once + $gcovfiles{$gcovfile}->{"seen"}=1; + + print "Getting coverage data from $gcovfile\n" if $debug; + + open( FH, "< $gcovfile" ) || die "Can't open $gcovfile for reading:$!\n"; + while() + { + my ($cov, $line, @code ) = split( /:/, $_ ); + $cov =~ s/^\s+//; # Strip leading space + $line =~ s/^\s+//; + my $code=join(":", @code); + if($cov =~ /\#/) + { + # There is no coverage data for these executable lines + $gcovfiles{$gcovfile}->{"uncovered"}->{$line}++; + $gcovfiles{$gcovfile}->{"src"}->{$line}=$code; + } + elsif( $cov ne "-" && looks_like_number($cov) && $cov > 0 ) + { + $gcovfiles{$gcovfile}->{"covered"}->{$line}=$cov; + $gcovfiles{$gcovfile}->{"src"}->{$line}=$code; + } + else + { + # All other lines are not executable. + $gcovfiles{$gcovfile}->{"src"}->{$line}=$code; + } + } + close( FH ); + } + $filesref->{$file}->{"coverage"} = $gcovfiles{$gcovfile}; # store hashref + } + else + { + # No gcov output - the gcno file produced no coverage of the source/header + # Probably means that there is no coverage for the file (with the given + # test case - there may be some somewhere, but for the sake of speed, don't + # check (yet). + } + } +} + +# Run the git diff command to get the patch, then check the coverage +# output for the patch. +sub run_diff +{ + my ($fh, $c) = $repo->command_output_pipe(@_); + our @patch=(); + while(<$fh>) + { + chomp; + push @patch, $_; + } + $repo->command_close_pipe($fh, $c); + + # @patch has slurped diff for all files... + my $filesref = parse_diff ( \@patch ); + show_patch_lines($filesref) if $debug; + + print "Checking coverage:\n" if $debug; + + my $cwd=getcwd(); + chdir ".cov" || die "Can't find $cwd/.cov:$!\n"; + + for my $file (keys(%$filesref)) + { + my ($name, $path, $suffix) = fileparse($file, qr{\.[^.]*$}); + if($suffix eq ".cpp" || $suffix eq ".c" || $suffix eq ".h") + { + get_coverage($file, $filesref); + } + } + chdir $cwd; + return $filesref; +} + + +sub calc_patch_coverage_percentage +{ + my $filesref = shift; + my $total_covered_lines = 0; + my $total_uncovered_lines = 0; + + foreach my $file (keys(%$filesref)) + { + my $covered_lines = 0; + my $uncovered_lines = 0; + + my $patchref = $filesref->{$file}->{"patch"}; + my $coverage_ref = $filesref->{$file}->{"coverage"}; + if( $coverage_ref ) + { + for my $patch (@$patchref) + { + for(my $i = 0; $i < $patch->[1]; $i++ ) + { + my $line = $i + $patch->[0]; + if($coverage_ref->{"covered"}->{$line}) + { + $covered_lines++; + $total_covered_lines++; + } + if($coverage_ref->{"uncovered"}->{$line}) + { + $uncovered_lines++; + $total_uncovered_lines++; + } + } + } + $coverage_ref->{"covered_lines"} = $covered_lines; + $coverage_ref->{"uncovered_lines"} = $uncovered_lines; + my $total = $covered_lines + $uncovered_lines; + my $percent = 0; + if($total > 0) + { + $percent = $covered_lines / $total; + } + $coverage_ref->{"percent_covered"} = 100 * $percent; + } + } + my $total_exec = $total_covered_lines + $total_uncovered_lines; + my $percent = 0; + if($total_exec > 0) { $percent = 100 * $total_covered_lines / $total_exec; } + + return $percent; +} + +sub patch_output +{ + my $filesref = shift; + foreach my $file (keys(%$filesref)) + { + my ($name, $path, $suffix) = fileparse($file, qr{\.[^.]*$}); + next if($path !~ /^dali/); + + my $patchref = $filesref->{$file}->{"patch"}; + my $b_lines_ref = $filesref->{$file}->{"b_lines"}; + my $coverage_ref = $filesref->{$file}->{"coverage"}; + print BOLD, "$file "; + + if($coverage_ref) + { + if( $coverage_ref->{"covered_lines"} > 0 + || + $coverage_ref->{"uncovered_lines"} > 0 ) + { + print GREEN, "Covered: " . $coverage_ref->{"covered_lines"}, RED, " Uncovered: " . $coverage_ref->{"uncovered_lines"}, RESET; + } + } + else + { + if($suffix eq ".cpp" || $suffix eq ".c" || $suffix eq ".h") + { + print RED; + } + print "No coverage found"; + } + print RESET "\n"; + + for my $patch (@$patchref) + { + my $hunkstr="Hunk: " . $patch->[0]; + if( $patch->[1] > 1 ) + { + $hunkstr .= " - " . ($patch->[0]+$patch->[1]-1); + } + print BOLD, "$hunkstr\n", RESET; + for(my $i = 0; $i < $patch->[1]; $i++ ) + { + my $line = $i + $patch->[0]; + printf "%-6s ", $line; + + if($coverage_ref) + { + my $color; + if($coverage_ref->{"covered"}->{$line}) + { + $color=GREEN; + } + elsif($coverage_ref->{"uncovered"}->{$line}) + { + $color=BOLD . RED; + } + else + { + $color=BLACK; + } + my $src=$coverage_ref->{"src"}->{$line}; + chomp($src); + print $color, "$src\n", RESET; + } + else + { + # We don't have coverage data, so print it from the patch instead. + my $src = $b_lines_ref->{$line}; + print "$src\n"; + } + } + } + } +} + + +################################################################################ +## MAIN ## +################################################################################ + +my $cwd = getcwd(); +chdir $repo->wc_path(); +chdir "build/tizen"; +`make rename_cov_data`; + +my @cmd=('--no-pager','diff','--no-ext-diff','-U0','--no-color'); + +my $status = $repo->command("status", "-s"); +if( $status eq "" ) +{ + # There are no changes in the index or working tree. Use the last patch instead + push @cmd, ('HEAD~1','HEAD'); +} +elsif($opt_cached) # TODO: Remove this option. Instead, need full diff +{ + push @cmd, "--cached"; +} + +push @cmd, @ARGV; +my $filesref = run_diff(@cmd); + +my $percent = calc_patch_coverage_percentage($filesref); +if( ! $opt_quiet ) +{ + patch_output($filesref); + my $color=BOLD RED; + if($percent>=90) + { + $color=GREEN; + } + printf("Percentage of change covered: $color %5.2f%\n" . RESET, $percent); +} +exit($percent<90); + + +__END__ + +=head1 NAME + +patch-coverage + +=head1 SYNOPSIS + +patch-coverage.pl - Determine if patch coverage is below 90% + +=head1 DESCRIPTION +Calculates how well the most recent patch is covered (either the +patch that is in the index+working directory, or HEAD). + +=head1 OPTIONS + +=over 28 + +=item B<-c|--cached> +Use index files if there is nothing in the working tree + +=item B< --help> +This help + +=item B<-q|--quiet> +Don't generate any output + +=head1 RETURN STATUS +0 if the coverage of source files is > 90%, otherwise 1 + +=head1 EXAMPLES + + +=cut diff --git a/build/tizen/Makefile.am b/build/tizen/Makefile.am index 925f052d7..7d9bf9511 100644 --- a/build/tizen/Makefile.am +++ b/build/tizen/Makefile.am @@ -40,7 +40,7 @@ COVERAGE_OUTPUT_DIR=doc/coverage # From lcov version 1.10 onwards, branch coverage is off by default and earlier versions do not support the rc option LCOV_OPTS=`if [ \`printf "\\\`lcov --version | cut -d' ' -f4\\\`\n1.10\n" | sort -V | head -n 1\` = 1.10 ] ; then echo "--rc lcov_branch_coverage=1" ; fi` -cov_data: +rename_cov_data: @test -z $(COVERAGE_DIR) || mkdir -p $(COVERAGE_DIR) @rm -f $(COVERAGE_DIR)/* @cp dali-core/.libs/*.gcda dali-core/.libs/*.gcno $(COVERAGE_DIR) @@ -48,6 +48,8 @@ cov_data: do mv $$i `echo $$i | sed s/libdali_core_la-//` ; echo $$i ; done @for i in `find $(COVERAGE_DIR) -name "libdali_la-*.gcda" -o -name "libdali_la-*.gcno"` ;\ do mv $$i `echo $$i | sed s/libdali_la-//` ; done + +cov_data: rename_cov_data @cd $(COVERAGE_DIR) ; lcov $(LCOV_OPTS) --base-directory . --directory . -c -o dali.info @cd $(COVERAGE_DIR) ; lcov $(LCOV_OPTS) --remove dali.info "*/dali-env/*" "/usr/include/*" "public-api/shader-effects/*" "*/image-actor*" -o dali.info