Tests/RunCMake/RunCMake.cmake

   1 foreach(arg
   2     RunCMake_GENERATOR
   3     RunCMake_SOURCE_DIR
   4     RunCMake_BINARY_DIR
   5     )
   6   if(NOT DEFINED ${arg})
   7     message(FATAL_ERROR "${arg} not given!")
   8   endif()
   9 endforeach()
  10
  11 function(run_cmake test)
  12   if(DEFINED ENV{RunCMake_TEST_FILTER} AND NOT test MATCHES "$ENV{RunCMake_TEST_FILTER}")
  13     return()
  14   endif()
  15
  16   set(top_src "${RunCMake_SOURCE_DIR}")
  17   set(top_bin "${RunCMake_BINARY_DIR}")
  18   if(EXISTS ${top_src}/${test}-result.txt)
  19     file(READ ${top_src}/${test}-result.txt expect_result)
  20     string(REGEX REPLACE "\n+$" "" expect_result "${expect_result}")
  21   else()
  22     set(expect_result 0)
  23   endif()
  24
  25   string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} platform_name)
  26   #remove all additional bits from cygwin/msys name
  27   if(platform_name MATCHES cygwin)
  28     set(platform_name cygwin)
  29   endif()
  30   if(platform_name MATCHES msys)
  31     set(platform_name msys)
  32   endif()
  33
  34   foreach(o out err)
  35     if(RunCMake-std${o}-file AND EXISTS ${top_src}/${RunCMake-std${o}-file})
  36       file(READ ${top_src}/${RunCMake-std${o}-file} expect_std${o})
  37       string(REGEX REPLACE "\n+$" "" expect_std${o} "${expect_std${o}}")
  38     elseif(EXISTS ${top_src}/${test}-std${o}-${platform_name}.txt)
  39       file(READ ${top_src}/${test}-std${o}-${platform_name}.txt expect_std${o})
  40       string(REGEX REPLACE "\n+$" "" expect_std${o} "${expect_std${o}}")
  41     elseif(EXISTS ${top_src}/${test}-std${o}.txt)
  42       file(READ ${top_src}/${test}-std${o}.txt expect_std${o})
  43       string(REGEX REPLACE "\n+$" "" expect_std${o} "${expect_std${o}}")
  44     else()
  45       unset(expect_std${o})
  46     endif()
  47   endforeach()
  48   if (NOT expect_stderr)
  49     if (NOT RunCMake_DEFAULT_stderr)
  50       set(RunCMake_DEFAULT_stderr "^$")
  51     endif()
  52     set(expect_stderr ${RunCMake_DEFAULT_stderr})
  53   endif()
  54
  55   if (NOT RunCMake_TEST_SOURCE_DIR)
  56     set(RunCMake_TEST_SOURCE_DIR "${top_src}")
  57   endif()
  58   if(NOT RunCMake_TEST_BINARY_DIR)
  59     set(RunCMake_TEST_BINARY_DIR "${top_bin}/${test}-build")
  60   endif()
  61   if(NOT RunCMake_TEST_NO_CLEAN)
  62     file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
  63   endif()
  64   file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
  65   if(RunCMake-prep-file AND EXISTS ${top_src}/${RunCMake-prep-file})
  66     include(${top_src}/${RunCMake-prep-file})
  67   else()
  68     include(${top_src}/${test}-prep.cmake OPTIONAL)
  69   endif()
  70   if(RunCMake_TEST_OUTPUT_MERGE)
  71     set(actual_stderr_var actual_stdout)
  72     set(actual_stderr "")
  73   else()
  74     set(actual_stderr_var actual_stderr)
  75   endif()
  76   if(DEFINED RunCMake_TEST_TIMEOUT)
  77     set(maybe_timeout TIMEOUT ${RunCMake_TEST_TIMEOUT})
  78   else()
  79     set(maybe_timeout "")
  80   endif()
  81   if(RunCMake-stdin-file AND EXISTS ${top_src}/${RunCMake-stdin-file})
  82     set(maybe_input_file INPUT_FILE ${top_src}/${RunCMake-stdin-file})
  83   elseif(EXISTS ${top_src}/${test}-stdin.txt)
  84     set(maybe_input_file INPUT_FILE ${top_src}/${test}-stdin.txt)
  85   else()
  86     set(maybe_input_file "")
  87   endif()
  88   if(NOT RunCMake_TEST_COMMAND)
  89     if(NOT DEFINED RunCMake_TEST_OPTIONS)
  90       set(RunCMake_TEST_OPTIONS "")
  91     endif()
  92     if(APPLE)
  93       list(APPEND RunCMake_TEST_OPTIONS -DCMAKE_POLICY_DEFAULT_CMP0025=NEW)
  94     endif()
  95     if(NOT RunCMake_TEST_NO_CMP0129 AND CMAKE_C_COMPILER_ID STREQUAL "LCC")
  96       list(APPEND RunCMake_TEST_OPTIONS -DCMAKE_POLICY_DEFAULT_CMP0129=NEW)
  97     endif()
  98     if(RunCMake_MAKE_PROGRAM)
  99       list(APPEND RunCMake_TEST_OPTIONS "-DCMAKE_MAKE_PROGRAM=${RunCMake_MAKE_PROGRAM}")
 100     endif()
 101     set(RunCMake_TEST_COMMAND ${CMAKE_COMMAND})
 102     if(NOT RunCMake_TEST_NO_SOURCE_DIR)
 103       list(APPEND RunCMake_TEST_COMMAND "${RunCMake_TEST_SOURCE_DIR}")
 104     endif()
 105     list(APPEND RunCMake_TEST_COMMAND -G "${RunCMake_GENERATOR}")
 106     if(RunCMake_GENERATOR_PLATFORM)
 107       list(APPEND RunCMake_TEST_COMMAND -A "${RunCMake_GENERATOR_PLATFORM}")
 108     endif()
 109     if(RunCMake_GENERATOR_TOOLSET)
 110       list(APPEND RunCMake_TEST_COMMAND -T "${RunCMake_GENERATOR_TOOLSET}")
 111     endif()
 112     if(RunCMake_GENERATOR_INSTANCE)
 113       list(APPEND RunCMake_TEST_COMMAND "-DCMAKE_GENERATOR_INSTANCE=${RunCMake_GENERATOR_INSTANCE}")
 114     endif()
 115     list(APPEND RunCMake_TEST_COMMAND
 116       -DRunCMake_TEST=${test}
 117       --no-warn-unused-cli
 118       )
 119   else()
 120     set(RunCMake_TEST_OPTIONS "")
 121   endif()
 122   if(NOT DEFINED RunCMake_TEST_RAW_ARGS)
 123     set(RunCMake_TEST_RAW_ARGS "")
 124   endif()
 125   if(NOT RunCMake_TEST_COMMAND_WORKING_DIRECTORY)
 126     set(RunCMake_TEST_COMMAND_WORKING_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
 127   endif()
 128   string(CONCAT _code [[execute_process(
 129     COMMAND ${RunCMake_TEST_COMMAND}
 130             ${RunCMake_TEST_OPTIONS}
 131             ]] "${RunCMake_TEST_RAW_ARGS}\n" [[
 132     WORKING_DIRECTORY "${RunCMake_TEST_COMMAND_WORKING_DIRECTORY}"
 133     OUTPUT_VARIABLE actual_stdout
 134     ERROR_VARIABLE ${actual_stderr_var}
 135     RESULT_VARIABLE actual_result
 136     ENCODING UTF8
 137     ${maybe_timeout}
 138     ${maybe_input_file}
 139     )]])
 140   cmake_language(EVAL CODE "${_code}")
 141   set(msg "")
 142   if(NOT "${actual_result}" MATCHES "${expect_result}")
 143     string(APPEND msg "Result is [${actual_result}], not [${expect_result}].\n")
 144   endif()
 145
 146   # Special case: remove ninja no-op line from stderr, but not stdout.
 147   # Test cases that look for it should use RunCMake_TEST_OUTPUT_MERGE.
 148   string(REGEX REPLACE "(^|\r?\n)ninja: no work to do\\.\r?\n" "\\1" actual_stderr "${actual_stderr}")
 149
 150   # Remove incidental content from both stdout and stderr.
 151   string(CONCAT ignore_line_regex
 152     "(^|\n)((==[0-9]+=="
 153     "|BullseyeCoverage"
 154     "|[a-z]+\\([0-9]+\\) malloc:"
 155     "|clang[^:]*: warning: the object size sanitizer has no effect at -O0, but is explicitly enabled:"
 156     "|Error kstat returned"
 157     "|Hit xcodebuild bug"
 158     "|Recompacting log\\.\\.\\."
 159
 160     "|LICENSE WARNING:"
 161     "|Your license to use PGI[^\n]*expired"
 162     "|Please obtain a new version at"
 163     "|contact PGI Sales at"
 164
 165     "|[^\n]*install_name_tool: warning: changes being made to the file will invalidate the code signature in:"
 166     "|[^\n]*xcodebuild[^\n]*DVTPlugInManager"
 167     "|[^\n]*xcodebuild[^\n]*Requested but did not find extension point with identifier"
 168     "|[^\n]*xcodebuild[^\n]*warning: file type[^\n]*is based on missing file type"
 169     "|[^\n]*objc[^\n]*: Class [^\n]* One of the two will be used. Which one is undefined."
 170     "|[^\n]*is a member of multiple groups"
 171     "|[^\n]*offset in archive not a multiple of 8"
 172     "|[^\n]*from Time Machine by path"
 173     "|[^\n]*Bullseye Testing Technology"
 174     ")[^\n]*\n)+"
 175     )
 176   foreach(o out err)
 177     string(REGEX REPLACE "\r\n" "\n" actual_std${o} "${actual_std${o}}")
 178     string(REGEX REPLACE "${ignore_line_regex}" "\\1" actual_std${o} "${actual_std${o}}")
 179     string(REGEX REPLACE "\n+$" "" actual_std${o} "${actual_std${o}}")
 180     set(expect_${o} "")
 181     if(DEFINED expect_std${o})
 182       if(NOT "${actual_std${o}}" MATCHES "${expect_std${o}}")
 183         string(REGEX REPLACE "\n" "\n expect-${o}> " expect_${o}
 184           " expect-${o}> ${expect_std${o}}")
 185         set(expect_${o} "Expected std${o} to match:\n${expect_${o}}\n")
 186         string(APPEND msg "std${o} does not match that expected.\n")
 187       endif()
 188     endif()
 189   endforeach()
 190   unset(RunCMake_TEST_FAILED)
 191   if(RunCMake-check-file AND EXISTS ${top_src}/${RunCMake-check-file})
 192     include(${top_src}/${RunCMake-check-file})
 193   else()
 194     include(${top_src}/${test}-check.cmake OPTIONAL)
 195   endif()
 196   if(RunCMake_TEST_FAILED)
 197     set(msg "${RunCMake_TEST_FAILED}\n${msg}")
 198   endif()
 199   if(msg)
 200     string(REPLACE ";" "\" \"" command "\"${RunCMake_TEST_COMMAND}\"")
 201     if(RunCMake_TEST_OPTIONS)
 202       string(REPLACE ";" "\" \"" options "\"${RunCMake_TEST_OPTIONS}\"")
 203       string(APPEND command " ${options}")
 204     endif()
 205     if(RunCMake_TEST_RAW_ARGS)
 206       string(APPEND command " ${RunCMake_TEST_RAW_ARGS}")
 207     endif()
 208     string(APPEND msg "Command was:\n command> ${command}\n")
 209   endif()
 210   if(msg)
 211     string(REGEX REPLACE "\n" "\n actual-out> " actual_out " actual-out> ${actual_stdout}")
 212     string(REGEX REPLACE "\n" "\n actual-err> " actual_err " actual-err> ${actual_stderr}")
 213     message(SEND_ERROR "${test}${RunCMake_TEST_VARIANT_DESCRIPTION} - FAILED:\n"
 214       "${msg}"
 215       "${expect_out}"
 216       "Actual stdout:\n${actual_out}\n"
 217       "${expect_err}"
 218       "Actual stderr:\n${actual_err}\n"
 219       )
 220   else()
 221     message(STATUS "${test}${RunCMake_TEST_VARIANT_DESCRIPTION} - PASSED")
 222   endif()
 223 endfunction()
 224
 225 function(run_cmake_command test)
 226   set(RunCMake_TEST_COMMAND "${ARGN}")
 227   run_cmake(${test})
 228 endfunction()
 229
 230 function(run_cmake_script test)
 231   set(RunCMake_TEST_COMMAND ${CMAKE_COMMAND} ${ARGN} -P ${RunCMake_SOURCE_DIR}/${test}.cmake)
 232   run_cmake(${test})
 233 endfunction()
 234
 235 function(run_cmake_with_options test)
 236   set(RunCMake_TEST_OPTIONS "${ARGN}")
 237   run_cmake(${test})
 238 endfunction()
 239
 240 function(run_cmake_with_raw_args test args)
 241   set(RunCMake_TEST_RAW_ARGS "${args}")
 242   run_cmake(${test})
 243 endfunction()
 244
 245 function(ensure_files_match expected_file actual_file)
 246   if(NOT EXISTS "${expected_file}")
 247     message(FATAL_ERROR "Expected file does not exist:\n  ${expected_file}")
 248   endif()
 249   if(NOT EXISTS "${actual_file}")
 250     message(FATAL_ERROR "Actual file does not exist:\n  ${actual_file}")
 251   endif()
 252   file(READ "${expected_file}" expected_file_content)
 253   file(READ "${actual_file}" actual_file_content)
 254   if(NOT "${expected_file_content}" STREQUAL "${actual_file_content}")
 255     message(FATAL_ERROR "Actual file content does not match expected:\n
 256     \n
 257       expected file: ${expected_file}\n
 258       expected content:\n
 259       ${expected_file_content}\n
 260     \n
 261       actual file: ${actual_file}\n
 262       actual content:\n
 263       ${actual_file_content}\n
 264     ")
 265   endif()
 266 endfunction()
 267
 268 # Protect RunCMake tests from calling environment.
 269 unset(ENV{MAKEFLAGS})