js/src/tests/test.sh

   1 #!/bin/bash -e
   2 # -*- Mode: Shell-script; tab-width: 4; indent-tabs-mode: nil; -*-
   3
   4 # ***** BEGIN LICENSE BLOCK *****
   5 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   6 #
   7 # The contents of this file are subject to the Mozilla Public License Version
   8 # 1.1 (the "License"); you may not use this file except in compliance with
   9 # the License. You may obtain a copy of the License at
  10 # http://www.mozilla.org/MPL/
  11 #
  12 # Software distributed under the License is distributed on an "AS IS" basis,
  13 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  14 # for the specific language governing rights and limitations under the
  15 # License.
  16 #
  17 # The Original Code is Mozilla JavaScript Testing Utilities
  18 #
  19 # The Initial Developer of the Original Code is
  20 # Mozilla Corporation.
  21 # Portions created by the Initial Developer are Copyright (C) 2007
  22 # the Initial Developer. All Rights Reserved.
  23 #
  24 # Contributor(s): Bob Clary <bclary@bclary.com>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 if [[ -z "$TEST_DIR" ]]; then
  41     cat <<EOF
  42 `basename $0`: error
  43
  44 TEST_DIR, the location of the Sisyphus framework,
  45 is required to be set prior to calling this script.
  46 EOF
  47     exit 2
  48 fi
  49
  50 if [[ ! -e $TEST_DIR/bin/library.sh ]]; then
  51     echo "TEST_DIR=$TEST_DIR"
  52     echo ""
  53     echo "This script requires the Sisyphus testing framework. Please "
  54     echo "cvs check out the Sisyphys framework from mozilla/testing/sisyphus"
  55     echo "and set the environment variable TEST_DIR to the directory where it"
  56     echo "located."
  57     echo ""
  58
  59     exit 2
  60 fi
  61
  62 source $TEST_DIR/bin/library.sh
  63
  64 TEST_JSDIR=${TEST_JSDIR:-$TEST_DIR/tests/mozilla.org/js}
  65
  66 TEST_JSSHELL_TIMEOUT=${TEST_JSSHELL_TIMEOUT:-480}
  67 TEST_JSEACH_TIMEOUT=${TEST_JSEACH_TIMEOUT:-485}
  68 TEST_JSEACH_PAGE_TIMEOUT=${TEST_JSEACH_PAGE_TIMEOUT:-480}
  69 TEST_JSALL_TIMEOUT=${TEST_JSALL_TIMEOUT:-21600}
  70 TEST_WWW_JS=`echo $TEST_JSDIR|sed "s|$TEST_DIR||"`
  71
  72 #
  73 # options processing
  74 #
  75 usage()
  76 {
  77     cat <<EOF
  78 usage: test.sh -p product -b branch -T buildtype -x executablepath -N profilename \\
  79                [-X excludetests] [-I includetests] [-c] [-t] [-F] [-d datafiles]
  80
  81 variable            description
  82 ===============     ============================================================
  83 -p product          required. firefox|thunderbird|js|fennec
  84 -b branch           required. one of supported branches. see library.sh.
  85 -s jsshellsourcepath       required for shell. path to js shell source directory mozilla/js/src
  86 -T buildtype        required. one of opt debug
  87 -x executablepath   required for browser. directory-tree containing executable 'product'
  88 -N profilename      required for browser. profile name
  89 -X excludetests     optional. By default the test will exclude the
  90                     tests listed in spidermonkey-n-\$branch.tests,
  91                     performance-\$branch.tests. excludetests is a list of either
  92                     individual tests, manifest files or sub-directories which
  93                     will override the default exclusion list.
  94 -I includetests     optional. By default the test will include the
  95                     JavaScript tests appropriate for the branch. includetests is a
  96                     list of either individual tests, manifest files or
  97                     sub-directories which will override the default inclusion
  98                     list.
  99 -c                  optional. By default the test will exclude tests
 100                     which crash on this branch, test type, build type and
 101                     operating system. -c will include tests which crash.
 102                     Typically this should only be used in combination with -R.
 103                     This has no effect on shell based tests which execute crash
 104                     tests regardless.
 105 -t                  optional. By default the test will exclude tests
 106                     which time out on this branch, test type, build type and
 107                     operating system. -t will include tests which timeout.
 108 -J jsoptions        optional. Set JavaScript options:
 109                     -Z n Set gczeal to n. Currently, only valid for
 110                     debug builds of Gecko 1.8.1.15, 1.9.0 and later.
 111                     -z optional. use split objects in the shell.
 112                     -j optional. use JIT in the shell. Only available on 1.9.1 and later
 113 -F                  optional. Just generate file lists without running any tests.
 114 -d datafiles        optional. one or more filenames of files containing
 115                     environment variable definitions to be included.
 116
 117                     note that the environment variables should have the same
 118                     names as in the "variable" column.
 119
 120 if an argument contains more than one value, it must be quoted.
 121 EOF
 122     exit 2
 123 }
 124
 125 while getopts "p:b:s:T:x:N:d:X:I:J:RctF" optname
 126 do
 127     case $optname in
 128         p)
 129             product=$OPTARG;;
 130         b)
 131             branch=$OPTARG;;
 132         T)
 133             buildtype=$OPTARG;;
 134         s)
 135             jsshellsourcepath=$OPTARG;;
 136         N)
 137             profilename=$OPTARG;;
 138         x)
 139             executablepath=$OPTARG;;
 140         X)
 141             excludetests=$OPTARG;;
 142         I)
 143             includetests=$OPTARG;;
 144         c)
 145             crashes=1;;
 146         t)
 147             timeouts=1;;
 148         F)
 149             filesonly=1;;
 150         J)
 151             javascriptoptions=$OPTARG
 152             ;;
 153         d) datafiles=$OPTARG;;
 154     esac
 155 done
 156
 157 if [[ -n "$javascriptoptions" ]]; then
 158     unset OPTIND
 159     while getopts "Z:jz" optname $javascriptoptions; do
 160         case $optname in
 161             Z)
 162                 gczealshell="-Z $OPTARG"
 163                 gczealbrowser=";gczeal=$OPTARG"
 164                 ;;
 165             z)
 166                 splitobjects="-z"
 167                 ;;
 168             j)
 169                 jitshell="-j"
 170                 jitbrowser=";jit"
 171                 ;;
 172         esac
 173     done
 174 fi
 175
 176 # include environment variables
 177 if [[ -n "$datafiles" ]]; then
 178     for datafile in $datafiles; do
 179         source $datafile
 180     done
 181 fi
 182
 183 if [[ -n "$gczeal" && "$buildtype" != "debug" ]]; then
 184     error "gczeal is supported for buildtype debug and not $buildtype"
 185 fi
 186
 187 dumpvars product branch buildtype jsshellsourcepath profilename executablepath excludetests includetests crashes timeouts filesonly javascriptoptions datafiles | sed "s|^|arguments: |"
 188
 189 pushd $TEST_JSDIR
 190
 191 case $product in
 192     js)
 193         if [[ -z "$branch" || -z "$buildtype"  || -z "$jsshellsourcepath" ]]; then
 194             usage
 195         fi
 196         source config.sh
 197         testtype=shell
 198         executable="$jsshellsourcepath/$JS_OBJDIR/js$EXE_EXT"
 199         ;;
 200
 201     firefox|thunderbird|fennec)
 202         if [[ -z "$branch" || -z "$buildtype"  || -z "$executablepath" || -z "$profilename" ]]; then
 203             usage
 204         fi
 205         testtype=browser
 206         executable=`get_executable $product $branch $executablepath`
 207
 208         ;;
 209     *)
 210         echo "Unknown product: $product"
 211         usage
 212         ;;
 213 esac
 214
 215 function shellfileversion()
 216 {
 217     local jsfile=$1
 218     local version
 219
 220     case $jsfile in
 221         ecma/*)
 222             version=150;;
 223         ecma_2/*)
 224             version=150;;
 225         ecma_3/*)
 226             version=150;;
 227         ecma_3_1/*)
 228             version=180;;
 229         js1_1/*)
 230             version=150;;
 231         js1_2/*)
 232             version=150;;
 233         js1_3/*)
 234             version=150;;
 235         js1_4/*)
 236             version=150;;
 237         js1_5/*)
 238             version=150;;
 239         js1_6/*)
 240             version=160;;
 241         js1_7/*)
 242             version=170;;
 243         js1_8/*)
 244             version=180;;
 245         js1_8_1/*)
 246             version=180;;
 247         js1_9/*)
 248             version=190;;
 249         js2_0/*)
 250             version=200;;
 251         *)
 252             version=150;;
 253     esac
 254     echo $version
 255 }
 256
 257 function browserfileversion()
 258 {
 259     local jsfile=$1
 260     local version
 261
 262     case $jsfile in
 263         ecma/*)
 264             version=1.5;;
 265         ecma_2/*)
 266             version=1.5;;
 267         ecma_3/*)
 268             version=1.5;;
 269         ecma_3_1/*)
 270             version=1.8;;
 271         js1_1/*)
 272             version=1.1;;
 273         js1_2/*)
 274             version=1.5;;
 275         js1_3/*)
 276             version=1.5;;
 277         js1_4/*)
 278             version=1.5;;
 279         js1_5/*)
 280             version=1.5;;
 281         js1_6/*)
 282             version=1.6;;
 283         js1_7/*)
 284             version=1.7;;
 285         js1_8/*)
 286             version=1.8;;
 287         js1_8_1/*)
 288             version=1.8;;
 289         js1_9/*)
 290             version=1.9;;
 291         js2_0/*)
 292             version=2.0;;
 293         *)
 294             version=1.5;;
 295     esac
 296     echo $version
 297 }
 298
 299 rm -f finished-$branch-$testtype-$buildtype
 300
 301 if ! make failures.txt; then
 302     error "during make failures.txt" $LINENO
 303 fi
 304
 305 includetestsfile="included-$branch-$testtype-$buildtype.tests"
 306 rm -f $includetestsfile
 307 touch $includetestsfile
 308
 309 if [[ -z "$includetests" ]]; then
 310     # by default include tests appropriate for the branch
 311     includetests="e4x ecma ecma_2 ecma_3 js1_1 js1_2 js1_3 js1_4 js1_5 js1_6"
 312
 313     case "$branch" in
 314         1.8.0)
 315             ;;
 316         1.8.1)
 317             includetests="$includetests js1_7"
 318             ;;
 319         1.9.0)
 320             includetests="$includetests js1_7 js1_8"
 321             ;;
 322         1.9.1)
 323             includetests="$includetests js1_7 js1_8 ecma_3_1 js1_8_1"
 324             ;;
 325         1.9.2)
 326             includetests="$includetests js1_7 js1_8 ecma_3_1 js1_8_1"
 327             ;;
 328         1.9.3)
 329             includetests="$includetests js1_7 js1_8 ecma_3_1 js1_8_1"
 330             ;;
 331     esac
 332 fi
 333
 334 for i in $includetests; do
 335     if [[ -f "$i" ]]; then
 336         echo "# including $i" >> $includetestsfile
 337         if echo $i | grep -q '\.js$'; then
 338             echo $i >> $includetestsfile
 339         else
 340             cat $i >> $includetestsfile
 341         fi
 342     elif [[ -d "$i" ]]; then
 343         find $i -name '*.js' -print | egrep -v '(shell|browser|template|jsref|userhook.*|\.#.*)\.js' | sed 's/^\.\///' | sort >> $includetestsfile
 344     fi
 345 done
 346
 347 excludetestsfile="excluded-$branch-$testtype-$buildtype.tests"
 348 rm -f $excludetestsfile
 349 touch $excludetestsfile
 350
 351 if [[ -z "$excludetests" ]]; then
 352     excludetests="spidermonkey-n-$branch.tests performance-$branch.tests"
 353 fi
 354
 355 for e in $excludetests; do
 356     if [[ -f "$e" ]]; then
 357         echo "# excluding $e" >> $excludetestsfile
 358         if echo $e | grep -q '\.js$'; then
 359             echo $e >> $excludetestsfile
 360         else
 361             cat $e >> $excludetestsfile
 362         fi
 363     elif [[ -d "$e" ]]; then
 364         find $e -name '*.js' -print | egrep -v '(shell|browser|template|jsref|userhook.*|\.#.*)\.js' | sed 's/^\.\///' | sort >> $excludetestsfile
 365     fi
 366 done
 367
 368 if [[ -z "$TEST_MOZILLA_HG" ]]; then
 369     repo=CVS
 370 else
 371     repo=`basename $TEST_MOZILLA_HG`
 372 fi
 373 debug "repo=$repo"
 374
 375 pattern="TEST_BRANCH=($branch|[.][*]), TEST_REPO=($repo|[.][*]), TEST_BUILDTYPE=($buildtype|[.][*]), TEST_TYPE=($testtype|[.][*]), TEST_OS=($OSID|[.][*]), TEST_KERNEL=($TEST_KERNEL|[.][*]), TEST_PROCESSORTYPE=($TEST_PROCESSORTYPE|[.][*]), TEST_MEMORY=($TEST_MEMORY|[.][*]),"
 376
 377 if [[ -z "$timeouts" ]]; then
 378     echo "# exclude tests that time out" >> $excludetestsfile
 379     egrep "$pattern .*TEST_EXITSTATUS=[^,]*TIMED OUT[^,]*," failures.txt | \
 380         sed 's/.*TEST_ID=\([^,]*\),.*/\1/' | sort -u >> $excludetestsfile
 381 fi
 382
 383 if [[ -z "$crashes" ]]; then
 384     echo "# exclude tests that crash" >> $excludetestsfile
 385     egrep "$pattern .*TEST_EXITSTATUS=[^,]*(CRASHED|ABNORMAL)[^,]*" failures.txt  | \
 386         sed 's/.*TEST_ID=\([^,]*\),.*/\1/' | sort -u >> $excludetestsfile
 387
 388 fi
 389
 390 cat $includetestsfile | sed 's|^|include: |'
 391 cat $excludetestsfile | sed 's|^|exclude: |'
 392
 393 case $testtype in
 394     shell)
 395         echo "JavaScriptTest: Begin Run"
 396         cat $includetestsfile | while read jsfile
 397         do
 398             if echo $jsfile | grep -q '^#'; then
 399                 continue
 400             fi
 401
 402             if ! grep -q $jsfile $excludetestsfile; then
 403
 404                 version=`shellfileversion $jsfile`
 405
 406                 subsuitetestdir=`dirname $jsfile`
 407                 suitetestdir=`dirname $subsuitetestdir`
 408                 echo "JavaScriptTest: Begin Test $jsfile"
 409                 if [[ -z "$NARCISSUS" ]]; then
 410                     if eval $TIMECOMMAND timed_run.py $TEST_JSEACH_TIMEOUT \"$jsfile\" \
 411                         $EXECUTABLE_DRIVER \
 412                         $executable -v $version \
 413                         -S 524288 \
 414                         $gczealshell \
 415                         $splitobjects \
 416                         $jitshell \
 417                         -f ./shell.js \
 418                         -f $suitetestdir/shell.js \
 419                         -f $subsuitetestdir/shell.js \
 420                         -f ./$jsfile \
 421                         -f ./js-test-driver-end.js; then
 422                         true
 423                     else
 424                         rc=$?
 425                     fi
 426                 else
 427                     if eval $TIMECOMMAND timed_run.py $TEST_JSEACH_TIMEOUT \"$jsfile\" \
 428                         $EXECUTABLE_DRIVER \
 429                         $executable -v $version \
 430                         -S 524288 \
 431                         $gczealshell \
 432                         $splitobjects \
 433                         $jitshell \
 434                         -f $NARCISSUS \
 435                         -e "evaluate\(\'load\(\\\"./shell.js\\\"\)\'\)" \
 436                         -e "evaluate\(\'load\(\\\"$suitetestdir/shell.js\\\"\)\'\)" \
 437                         -e "evaluate\(\'load\(\\\"$subsuitetestdir/shell.js\\\"\)\'\)" \
 438                         -e "evaluate\(\'load\(\\\"./$jsfile\\\"\)\'\)" \
 439                         -e "evaluate\(\'load\(\\\"./js-test-driver-end.js\\\"\)\'\)"; then
 440                         true
 441                     else
 442                         rc=$?
 443                     fi
 444                 fi
 445                 if [[ $rc == 99 ]]; then
 446                     # note that this loop is executing in a sub-process
 447                     # error will terminate the sub-process but will transfer
 448                     # control to the next statement following the loop which
 449                     # in this case is the "End Run" output which incorrectly
 450                     # labels the test run as completed.
 451                     error "User Interrupt"
 452                 fi
 453                 echo "JavaScriptTest: End Test $jsfile"
 454             fi
 455         done
 456         rc=$?
 457         if [[ $rc != 0 ]]; then
 458             error ""
 459         fi
 460         echo "JavaScriptTest: End Run"
 461         ;;
 462
 463     browser)
 464         urllist="urllist-$branch-$testtype-$buildtype.tests"
 465         urlhtml="urllist-$branch-$testtype-$buildtype.html"
 466
 467         rm -f $urllist $urlhtml
 468
 469         cat > $urlhtml <<EOF
 470 <html xmlns="http://www.w3.org/1999/xhtml">
 471 <head>
 472 <title>JavaScript Tests</title>
 473 </head>
 474 <body>
 475 <ul>
 476 EOF
 477
 478         cat $includetestsfile | while read jsfile
 479         do
 480             if echo $jsfile | grep -q '^#'; then
 481                 continue
 482             fi
 483
 484             if ! grep -q $jsfile $excludetestsfile; then
 485
 486                 version=";version=`browserfileversion $jsfile`"
 487
 488                 echo "http://$TEST_HTTP/$TEST_WWW_JS/js-test-driver-standards.html?test=$jsfile;language=type;text/javascript$version$gczealbrowser$jitbrowser" >> $urllist
 489                 echo "<li><a href='http://$TEST_HTTP/$TEST_WWW_JS/js-test-driver-standards.html?test=$jsfile;language=type;text/javascript$version$gczealbrowser$jitbrowser'>$jsfile</a></li>" >> $urlhtml
 490             fi
 491         done
 492
 493         cat >> $urlhtml <<EOF
 494 </ul>
 495 </body>
 496 </html>
 497 EOF
 498
 499         chmod a+r $urlhtml
 500
 501         if [[ -z "$filesonly" ]]; then
 502             echo "JavaScriptTest: Begin Run"
 503             cat "$urllist" | while read url;
 504             do
 505                 edit-talkback.sh -p "$product" -b "$branch" -x "$executablepath" -i "$url"
 506                 jsfile=`echo $url | sed "s|http://$TEST_HTTP/$TEST_WWW_JS/js-test-driver-standards.html?test=\([^;]*\);.*|\1|"`
 507                 echo "JavaScriptTest: Begin Test $jsfile"
 508                 if eval $TIMECOMMAND timed_run.py $TEST_JSEACH_TIMEOUT \"$jsfile\" \
 509                     $EXECUTABLE_DRIVER \
 510                     \"$executable\" -P \"$profilename\" \
 511                     -spider -start -quit \
 512                     -uri \"$url\" \
 513                     -depth 0 -timeout \"$TEST_JSEACH_PAGE_TIMEOUT\" \
 514                     -hook \"http://$TEST_HTTP/$TEST_WWW_JS/userhookeach.js\"; then
 515                     true
 516                 else
 517                     rc=$?
 518                 fi
 519                 if [[ $rc == 99 ]]; then
 520                     error "User Interrupt"
 521                 fi
 522                 echo "JavaScriptTest: End Test $jsfile"
 523             done
 524             echo "JavaScriptTest: End Run"
 525         fi
 526         ;;
 527     *)
 528         ;;
 529 esac
 530
 531 popd