Tools/Scripts/run-bindings-tests

   1 #!/usr/bin/python
   2 # Copyright (C) 2010 Google Inc.  All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions
   6 # are met:
   7 # 1. Redistributions of source code must retain the above copyright
   8 #    notice, this list of conditions and the following disclaimer.
   9 # 2. Redistributions in binary form must reproduce the above copyright
  10 #    notice, this list of conditions and the following disclaimer in the
  11 #    documentation and/or other materials provided with the distribution.
  12 #
  13 # THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
  14 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
  17 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21 # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24 #
  25
  26 # This script generates h and cpp file for TestObj.idl using the V8 code
  27 # generator. Please execute the script whenever changes are made to
  28 # CodeGeneratorV8.pm, and submit the changes in V8TestObj.h/cpp in the same
  29 # patch. This makes it easier to track and review changes in generated code.
  30 # To execute, invoke: 'python run_tests.py'
  31
  32 import os
  33 import os.path
  34 import shutil
  35 import subprocess
  36 import sys
  37 import tempfile
  38 from webkitpy.common.checkout.scm.detection import detect_scm_system
  39
  40
  41 def generate_from_idl(generator, idl_file, output_directory, supplemental_dependency_file):
  42     cmd = ['perl', '-w',
  43            '-IWebCore/bindings/scripts',
  44            'WebCore/bindings/scripts/generate-bindings.pl',
  45            # idl include directories (path relative to generate-bindings.pl)
  46            '--include', '.',
  47            '--defines', 'TESTING_%s' % generator,
  48            '--generator', generator,
  49            '--outputDir', output_directory,
  50            '--supplementalDependencyFile', supplemental_dependency_file,
  51            idl_file]
  52
  53     process = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
  54     exit_code = process.wait()
  55     output = process.communicate()[0]
  56     if output:
  57         print output
  58     return exit_code
  59
  60
  61 def generate_supplemental_dependency(input_directory, supplemental_dependency_file):
  62     idl_files_list = tempfile.mkstemp()
  63     for input_file in os.listdir(input_directory):
  64         (name, extension) = os.path.splitext(input_file)
  65         if extension != '.idl':
  66             continue
  67         os.write(idl_files_list[0], os.path.join(input_directory, input_file) + "\n")
  68     os.close(idl_files_list[0])
  69
  70     cmd = ['perl', '-w',
  71            '-IWebCore/bindings/scripts',
  72            'WebCore/bindings/scripts/resolve-supplemental.pl',
  73            '--idlFilesList', idl_files_list[1],
  74            '--defines', '',
  75            '--supplementalDependencyFile', supplemental_dependency_file]
  76     process = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
  77     exit_code = process.wait()
  78     output = process.communicate()[0]
  79     if output:
  80         print output
  81
  82     os.remove(idl_files_list[1])
  83     return exit_code
  84
  85
  86 def detect_changes(generator, work_directory, reference_directory):
  87     changes_found = False
  88     for output_file in os.listdir(work_directory):
  89         cmd = ['diff',
  90                '-u',
  91                '-N',
  92                os.path.join(reference_directory, output_file),
  93                os.path.join(work_directory, output_file)]
  94
  95         process = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
  96         process.wait()
  97         output = process.communicate()[0]
  98         if output:
  99             print 'FAIL: (%s) %s' % (generator, output_file)
 100             print output
 101             changes_found = True
 102         else:
 103             print 'PASS: (%s) %s' % (generator, output_file)
 104
 105     return changes_found
 106
 107
 108 def run_tests(generator, input_directory, reference_directory, reset_results, supplemental_dependency_file):
 109     work_directory = reference_directory
 110
 111     passed = True
 112     for input_file in os.listdir(input_directory):
 113         (name, extension) = os.path.splitext(input_file)
 114         if extension != '.idl':
 115             continue
 116         # Generate output into the work directory (either the given one or a
 117         # temp one if not reset_results is performed)
 118         if not reset_results:
 119             work_directory = tempfile.mkdtemp()
 120
 121         if generate_from_idl(generator,
 122                              os.path.join(input_directory, input_file),
 123                              work_directory,
 124                              supplemental_dependency_file):
 125             passed = False
 126
 127         if reset_results:
 128             print "Reset results: (%s) %s" % (generator, input_file)
 129             continue
 130
 131         # Detect changes
 132         if detect_changes(generator, work_directory, reference_directory):
 133             passed = False
 134         shutil.rmtree(work_directory)
 135
 136     return passed
 137
 138
 139 def main(argv):
 140     """Runs WebCore bindings code generators on test IDL files and compares
 141     the results with reference files.
 142
 143     Options:
 144        --reset-results: Overwrites the reference files with the generated results.
 145
 146     """
 147     reset_results = "--reset-results" in argv
 148
 149     current_scm = detect_scm_system(os.curdir)
 150     os.chdir(os.path.join(current_scm.checkout_root, 'Source'))
 151
 152     all_tests_passed = True
 153
 154     generators = [
 155         'JS',
 156         'V8',
 157         'ObjC',
 158         'GObject',
 159         'CPP'
 160     ]
 161
 162     input_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test')
 163     supplemental_dependency_file = tempfile.mkstemp()[1]
 164     if generate_supplemental_dependency(input_directory, supplemental_dependency_file):
 165         print 'Failed to generate a supplemental dependency file.'
 166         os.remove(supplemental_dependency_file)
 167         return -1
 168
 169     for generator in generators:
 170         input_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test')
 171         reference_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test', generator)
 172         if not run_tests(generator, input_directory, reference_directory, reset_results, supplemental_dependency_file):
 173             all_tests_passed = False
 174
 175     os.remove(supplemental_dependency_file)
 176     print ''
 177     if all_tests_passed:
 178         print 'All tests PASS!'
 179         return 0
 180     else:
 181         print 'Some tests FAIL! (To update the reference files, execute "run-bindings-tests --reset-results")'
 182         return -1
 183
 184
 185 if __name__ == '__main__':
 186     sys.exit(main(sys.argv))