Improve reproducibility of test runs.

Add random seed to run-tests.py, using either a user supplied
value or a random number generated by random.SystemRandom().
This same random seed is passed to all test cases, making sure
that we can easily reproduce test failures that depend on
random numbers (i.e. bugs related to our handwritten ASLR).

Also fix all uses of rand() to make use of our RNG class
instead.

R=machenbach@chromium.org

Review URL: https://codereview.chromium.org/231443002

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@20637 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

diff --git a/test/cctest/test-strtod.cc b/test/cctest/test-strtod.cc
index 237d35db12d4d05db63ca27d45decc46ccb4c9f9..bebf4d14b06d37b0d690283a13cbe2d4fa22bf18 100644 (file)
--- a/test/cctest/test-strtod.cc
+++ b/test/cctest/test-strtod.cc
@@ -34,6 +34,7 @@
 #include "diy-fp.h"
 #include "double.h"
 #include "strtod.h"
+#include "utils/random-number-generator.h"
 
 using namespace v8::internal;
 
@@ -448,13 +449,13 @@ static const int kShortStrtodRandomCount = 2;
 static const int kLargeStrtodRandomCount = 2;
 
 TEST(RandomStrtod) {
-  srand(static_cast<unsigned int>(time(NULL)));
+  RandomNumberGenerator rng;
   char buffer[kBufferSize];
   for (int length = 1; length < 15; length++) {
     for (int i = 0; i < kShortStrtodRandomCount; ++i) {
       int pos = 0;
       for (int j = 0; j < length; ++j) {
-        buffer[pos++] = rand() % 10 + '0';
+        buffer[pos++] = rng.NextInt(10) + '0';
       }
       int exponent = DeterministicRandom() % (25*2 + 1) - 25 - length;
       buffer[pos] = '\0';
@@ -467,7 +468,7 @@ TEST(RandomStrtod) {
     for (int i = 0; i < kLargeStrtodRandomCount; ++i) {
       int pos = 0;
       for (int j = 0; j < length; ++j) {
-        buffer[pos++] = rand() % 10 + '0';
+        buffer[pos++] = rng.NextInt(10) + '0';
       }
       int exponent = DeterministicRandom() % (308*2 + 1) - 308 - length;
       buffer[pos] = '\0';

diff --git a/test/cctest/test-types.cc b/test/cctest/test-types.cc
index 3448a841bde55b9aadbf59b365f06364207792fc..7b727c71d9f9004e1568a4597d22c1ccb1f23a7b 100644 (file)
--- a/test/cctest/test-types.cc
+++ b/test/cctest/test-types.cc
@@ -29,6 +29,7 @@
 
 #include "cctest.h"
 #include "types.h"
+#include "utils/random-number-generator.h"
 
 using namespace v8::internal;
 
@@ -112,6 +113,8 @@ class Types {
     objects.push_back(array);
   }
 
+  RandomNumberGenerator rng;
+
   TypeHandle Representation;
   TypeHandle Semantic;
   TypeHandle None;
@@ -180,14 +183,14 @@ class Types {
   }
 
   TypeHandle Fuzz(int depth = 5) {
-    switch (rand() % (depth == 0 ? 3 : 20)) {
+    switch (rng.NextInt(depth == 0 ? 3 : 20)) {
       case 0: {  // bitset
         int n = 0
         #define COUNT_BITSET_TYPES(type, value) + 1
         BITSET_TYPE_LIST(COUNT_BITSET_TYPES)
         #undef COUNT_BITSET_TYPES
         ;
-        int i = rand() % n;
+        int i = rng.NextInt(n);
         #define PICK_BITSET_TYPE(type, value) \
           if (i-- == 0) return Type::type(region_);
         BITSET_TYPE_LIST(PICK_BITSET_TYPE)
@@ -195,13 +198,13 @@ class Types {
         UNREACHABLE();
       }
       case 1:  // class
-        switch (rand() % 2) {
+        switch (rng.NextInt(2)) {
           case 0: return ObjectClass;
           case 1: return ArrayClass;
         }
         UNREACHABLE();
       case 2:  // constant
-        switch (rand() % 6) {
+        switch (rng.NextInt(6)) {
           case 0: return SmiConstant;
           case 1: return Signed32Constant;
           case 2: return ObjectConstant1;
@@ -211,7 +214,7 @@ class Types {
         }
         UNREACHABLE();
       default: {  // union
-        int n = rand() % 10;
+        int n = rng.NextInt(10);
         TypeHandle type = None;
         for (int i = 0; i < n; ++i) {
           type = Type::Union(type, Fuzz(depth - 1), region_);

diff --git a/test/mjsunit/mjsunit.status b/test/mjsunit/mjsunit.status
index 28d3510711cc42df9675c03ff566194c92eb3e6c..9b9bf32ec0a32b8832bfa7f506f4f68eb92e2b04 100644 (file)
--- a/test/mjsunit/mjsunit.status
+++ b/test/mjsunit/mjsunit.status
@@ -93,7 +93,11 @@
   # This test sets the umask on a per-process basis and hence cannot be
   # used in multi-threaded runs.
   # On android there is no /tmp directory.
-  'd8-os': [PASS, ['isolates or arch == android_arm or arch == android_arm64 or arch == android_ia32', SKIP]],
+  # Currently d8-os generates a temporary directory name using Math.random(), so
+  # we cannot run several variants of d8-os simultaneously, since all of them
+  # get the same random seed and would generate the same directory name. Besides
+  # that, it doesn't make sense to run several variants of d8-os anyways.
+  'd8-os': [PASS, NO_VARIANTS, ['isolates or arch == android_arm or arch == android_arm64 or arch == android_ia32', SKIP]],
   'tools/tickprocessor': [PASS, ['arch == android_arm or arch == android_arm64 or arch == android_ia32', SKIP]],
 
   ##############################################################################

diff --git a/tools/run-tests.py b/tools/run-tests.py
index e6cb5a59ad6d34c70502a05b47c4206ec3e771e3..b1b86adc2b61c1d47f3194fc04d9815b7edbfa29 100755 (executable)
--- a/tools/run-tests.py
+++ b/tools/run-tests.py
@@ -34,6 +34,7 @@ import optparse
 import os
 from os.path import join
 import platform
+import random
 import shlex
 import subprocess
 import sys
@@ -200,6 +201,8 @@ def BuildOptions():
   result.add_option("--junittestsuite",
                     help="The testsuite name in the JUnit output file",
                     default="v8tests")
+  result.add_option("--random-seed", default=0, dest="random_seed",
+                    help="Default seed for initializing random generator")
   return result
 
 
@@ -250,6 +253,9 @@ def ProcessOptions(options):
   if options.j == 0:
     options.j = multiprocessing.cpu_count()
 
+  while options.random_seed == 0:
+    options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647)
+
   def excl(*args):
     """Returns true if zero or one of multiple arguments are true."""
     return reduce(lambda x, y: x + y, args) <= 1
@@ -396,7 +402,8 @@ def Execute(arch, mode, args, options, suites, workspace):
                         timeout, options.isolates,
                         options.command_prefix,
                         options.extra_flags,
-                        options.no_i18n)
+                        options.no_i18n,
+                        options.random_seed)
 
   # TODO(all): Combine "simulator" and "simulator_run".
   simulator_run = not options.dont_skip_simulator_slow_tests and \

diff --git a/tools/testrunner/local/execution.py b/tools/testrunner/local/execution.py
index 4453c08451f48cffd3891b4a62c7078e7d3972da..f4a40204e4889374fe9b94f9c0a628974dbd289d 100644 (file)
--- a/tools/testrunner/local/execution.py
+++ b/tools/testrunner/local/execution.py
@@ -171,6 +171,7 @@ class Runner(object):
     cmd = (self.context.command_prefix +
            [os.path.abspath(os.path.join(self.context.shell_dir, shell))] +
            d8testflag +
+           ["--random-seed=%s" % self.context.random_seed] +
            test.suite.GetFlagsForTestCase(test, self.context) +
            self.context.extra_flags)
     return cmd

diff --git a/tools/testrunner/objects/context.py b/tools/testrunner/objects/context.py
index 1f525b76b3114793b3548f2c740e7427805be0fc..68c19892410bf63643c17fe70c358d47cb69e327 100644 (file)
--- a/tools/testrunner/objects/context.py
+++ b/tools/testrunner/objects/context.py
@@ -28,7 +28,7 @@
 
 class Context():
   def __init__(self, arch, mode, shell_dir, mode_flags, verbose, timeout,
-               isolates, command_prefix, extra_flags, noi18n):
+               isolates, command_prefix, extra_flags, noi18n, random_seed):
     self.arch = arch
     self.mode = mode
     self.shell_dir = shell_dir
@@ -39,13 +39,16 @@ class Context():
     self.command_prefix = command_prefix
     self.extra_flags = extra_flags
     self.noi18n = noi18n
+    self.random_seed = random_seed
 
   def Pack(self):
     return [self.arch, self.mode, self.mode_flags, self.timeout, self.isolates,
-            self.command_prefix, self.extra_flags, self.noi18n]
+            self.command_prefix, self.extra_flags, self.noi18n,
+            self.random_seed]
 
   @staticmethod
   def Unpack(packed):
     # For the order of the fields, refer to Pack() above.
     return Context(packed[0], packed[1], None, packed[2], False,
-                   packed[3], packed[4], packed[5], packed[6], packed[7])
+                   packed[3], packed[4], packed[5], packed[6], packed[7],
+                   packed[8])