Enable ilasm round trip testing
authorKyungwoo Lee <kyulee@microsoft.com>
Wed, 30 Dec 2015 05:31:47 +0000 (21:31 -0800)
committerKyungwoo Lee <kyulee@microsoft.com>
Thu, 31 Dec 2015 18:16:02 +0000 (10:16 -0800)
This enables ilasm round-trip testing for Windows.
In order to test il{d}asm on xPlatforms, I injected ilasm round trip before executing the test.
This means this step is a part of the runtest not the buildtest -- unlike runtest, currently buildtest is only on Windows.
1. Test batch file is extended to run ildasm followed by ilasm.
2. BuildTest has now an optional flag "ilasmroundtrip" to enable the round-trip tests. Note under this flag, we run both the original binary and the round-trip binary respectively.
3. Test timeout is increased since the round-trip is part of runtime.
4. Did some clean-up on CLRTest.Execute.Batch.targets
5. Bail out the round-trip tests that are built against mscorlib instead of .Net Core.
6. Added /raweh to ildasm due to https://github.com/dotnet/coreclr/issues/2481
7. ilasmroundtrip.targets (~10 tests) are added to enable them by default in a regular test run.

Tested with enabling the entire tests (>5K tests) with 'ilasmroundtrip' on, which should be on in a daily or weekly stress mode.

14 files changed:
tests/buildtest.cmd
tests/src/CLRTest.Execute.Batch.targets
tests/src/IL.targets
tests/src/JIT/BBT/Scenario4/Not-Int32.ilproj
tests/src/JIT/CodeGenBringUpTests/Add1.csproj
tests/src/JIT/Directed/array-il/_Arrayscomplex3.ilproj
tests/src/JIT/Generics/Exceptions/general_class_instance01.csproj
tests/src/JIT/IL_Conformance/Old/directed/heap_ovf.ilproj
tests/src/JIT/Methodical/fp/apps/bouncingball_cs_d.csproj
tests/src/JIT/Performance/CodeQuality/BenchF/SqMtx/SqMtx.csproj
tests/src/JIT/Regression/CLR-x86-JIT/V1.2-M02/b00719/b00719.csproj
tests/src/JIT/SIMD/BitwiseOperations.csproj
tests/src/JIT/jit64/eh/basics/loopEH.csproj
tests/src/JIT/opt/perf/doublealign/Locals.csproj

index 0ce20a778bc5dbd64b437783c2e26736d36bae21..5e434a81a42a27d9980c1a538330ad5e99c06aa6 100644 (file)
@@ -33,6 +33,7 @@ if /i "%1" == "vs2013"   (set __VSVersion=%1&shift&goto Arg_Loop)
 if /i "%1" == "vs2015"   (set __VSVersion=%1&shift&goto Arg_Loop)
 
 if /i "%1" == "crossgen" (set _crossgen=true&shift&goto Arg_Loop)
+if /i "%1" == "ilasmroundtrip" (set _ilasmroundtrip=true&shift&goto Arg_Loop)
 if /i "%1" == "priority" (set _priorityvalue=%2&shift&shift&goto Arg_Loop)
 
 if /i "%1" == "verbose" (set _verbosity=detailed&shift&goto Arg_Loop)
@@ -43,6 +44,7 @@ goto Usage
 :ArgsDone
 
 if defined _crossgen echo Building tests with CrossGen enabled.&set _buildParameters=%_buildParameters% /p:CrossGen=true
+if defined _ilasmroundtrip echo Building tests with IlasmRoundTrip enabled.&set _buildParameters=%_buildParameters% /p:IlasmRoundTrip=true
 if defined _priorityvalue echo Building Test Priority %_priorityvalue%&set _buildParameters=%_buildParameters% /p:CLRTestPriorityToBuild=%_priorityvalue%
 if defined _verbosity echo Enabling verbose file logging
 if not defined _verbosity set _verbosity=normal
@@ -221,5 +223,6 @@ echo Clean - optional argument to force a clean build.
 echo VSVersion - optional argument to use VS2013 or VS2015  (default VS2015)
 echo CrossGen - Enables the tests to run crossgen on the test executables before executing them. 
 echo Priority (N) where N is a number greater than zero that signifies the set of tests that will be built and consequently run.
+echo IlasmRoundTrip - Enables ilasm round trip build and run of the tests before executing them.
 echo Verbose - Enables detailed file logging for the msbuild tasks.
 exit /b 1
index f8c27be279b62aad90f3ecad5a3ed12aaa451006..67805cc12018eed686a1a2650583373bf3b0c7bb 100644 (file)
@@ -46,6 +46,38 @@ if not exist "$(MSBuildProjectName).ni.exe" "%CORE_ROOT%\crossgen.exe" /Platform
     </PropertyGroup>
   </Target>
 
+  <Target
+    Name="GetIlasmRoundTripBatchScript"
+    Returns="$(IlasmRoundTripBatchScript)">
+    <PropertyGroup>
+      <InputAssemblyName Condition="'$(CLRTestKind)' == 'RunOnly'">$([MSBuild]::MakeRelative($(OutputPath), $(_CLRTestToRunFileFullPath)))</InputAssemblyName>
+      <InputAssemblyName Condition="'$(CLRTestKind)' == 'BuildAndRun'">$(MSBuildProjectName).exe</InputAssemblyName>
+      <DisassemblyName>$(MSBuildProjectName).dasm.il</DisassemblyName>
+      <TargetAssemblyName>$(MSBuildProjectName).asm.exe</TargetAssemblyName>
+
+      <!-- If a test is built against mscorlib instead of dotnet Core, permission attributes can be embedded, which CoreCLR does not support. -->
+      <IlasmRoundTrip Condition="'$(ReferenceLocalMscorlib)'!=''">false</IlasmRoundTrip>
+
+       <!-- https://github.com/dotnet/coreclr/issues/2481. Delete /raweh to ildasm once it is resolved.-->
+      <IlasmRoundTripBatchScript Condition="'$(IlasmRoundTrip)'=='true'">
+      <![CDATA[
+ECHO %CORE_ROOT%\ildasm.exe /raweh /out=$(DisassemblyName) $(InputAssemblyName)
+%CORE_ROOT%\ildasm.exe /raweh /out=$(DisassemblyName) $(InputAssemblyName)
+IF NOT "!ERRORLEVEL!"=="0" (
+  ECHO EXECUTION OF ILDASM - FAILED !ERRORLEVEL!
+  Exit /b 1
+)
+ECHO %CORE_ROOT%\ilasm.exe /output=$(TargetAssemblyName) $(_IlasmSwitches) $(DisassemblyName)
+%CORE_ROOT%\ilasm.exe /output=$(TargetAssemblyName) $(_IlasmSwitches) $(DisassemblyName)
+IF NOT "!ERRORLEVEL!"=="0" (
+  ECHO EXECUTION OF ILASM - FAILED !ERRORLEVEL!
+  Exit /b 1
+)
+      ]]>
+      </IlasmRoundTripBatchScript>
+    </PropertyGroup>
+  </Target>
+
   <!-- This is here because of this bug: http://blogs.msdn.com/b/msbuild/archive/2006/01/03/508629.aspx-->
   <Target Name="FetchExternalProperties">
     <!--Call GetExecuteShFullPath to get ToRunProject cmd file Path  -->
@@ -88,7 +120,7 @@ if not exist "$(MSBuildProjectName).ni.exe" "%CORE_ROOT%\crossgen.exe" /Platform
   <Target Name="GenerateBatchExecutionScript"
     Inputs="$(MSBuildProjectFullPath)"
     Outputs="$(OutputPath)\$(MSBuildProjectName).cmd"
-    DependsOnTargets="FetchExternalProperties;GetCrossgenBatchScript">
+    DependsOnTargets="FetchExternalProperties;GetCrossgenBatchScript;GetIlasmRoundTripBatchScript">
 
     <Message Text="Project depends on $(_CLRTestToRunFileFullPath)." Condition="'$(_CLRTestNeedsProjectToRun)' == 'True'" />
 
@@ -154,7 +186,7 @@ IF NOT "%CLRTestExitCode%"=="%CLRTestExpectedExitCode%" (
 
       <BatchCLRTestExecutionScriptArgument Include="coreroot">
         <HasParam>true</HasParam>
-        <ParamName>envScriptFullPath</ParamName>
+        <ParamName>CoreRootFullPath</ParamName>
         <Command><![CDATA[
     set CORE_ROOT=%2
         ]]></Command>
@@ -168,33 +200,24 @@ IF NOT "%CLRTestExitCode%"=="%CLRTestExpectedExitCode%" (
     </ItemGroup>
 
     <PropertyGroup> 
-      <_CLRTestExeFile Condition="'$(_CLRTestNeedsProjectToRun)' == 'true'">$(_CLRTestToRunFileFullPath)</_CLRTestExeFile>
-      <_CLRTestExeFile Condition="'$(_CLRTestNeedsProjectToRun)' == 'false'">$(MSBuildProjectName).exe</_CLRTestExeFile>
-      <_CLRTestExeFile Condition="'$(CLRTestIsHosted)'=='true' And '$(_CLRTestNeedsProjectToRun)'=='true'">$([MSBuild]::MakeRelative($(OutputPath), $(_CLRTestToRunFileFullPath)))</_CLRTestExeFile>
-      <_CLRTestRunFile Condition="'$(CLRTestIsHosted)'=='true'">"%CORE_ROOT%\corerun.exe" "$(_CLRTestExeFile)"</_CLRTestRunFile>
-      <_CLRTestRunFile Condition="'$(CLRTestIsHosted)'=='false'">"$(_CLRTestExeFile)"</_CLRTestRunFile>
+      <_CLRTestRunFile Condition="'$(CLRTestIsHosted)'=='true'">"%CORE_ROOT%\corerun.exe"</_CLRTestRunFile>
+      <BatchCLRTestLaunchCmds Condition=" '$(IlasmRoundTrip)'=='true' "><![CDATA[
+ECHO $(_CLRTestRunFile) $(TargetAssemblyName) %CLRTestExecutionArguments% %Host_Args%
+$(_CLRTestRunFile) $(TargetAssemblyName) %CLRTestExecutionArguments% %Host_Args%
 
-      <BatchCLRTestLaunchCmds Condition=" '$(BatchCLRTestLaunchCmds)'=='' "><![CDATA[
-IF NOT "%CLRCustomTestLauncher%"=="" (
-  goto :CustomLauncher
-) ELSE (
-  goto :DefaultLauncher
+IF NOT "!ERRORLEVEL!"=="%CLRTestExpectedExitCode%" (
+  ECHO END EXECUTION OF IL{D}ASM BINARY - FAILED !ERRORLEVEL! vs %CLRTestExpectedExitCode%
+  ECHO FAILED
+  Exit /b 1
 )
-
-:CustomLauncher
-  call %CLRCustomTestLauncher% %~dp0 $(_CLRTestExeFile) %CLRTestExecutionArguments% %Host_Args%
-  set CLRTestExitCode=%ERRORLEVEL%
-  goto :EndLauncher
-
-:DefaultLauncher
-  ECHO $(_CLRTestRunFile) %CLRTestExecutionArguments% %Host_Args%
-  %_DebuggerFullPath% $(_CLRTestRunFile) %CLRTestExecutionArguments% %Host_Args%
-  set CLRTestExitCode=%ERRORLEVEL%
-
-:EndLauncher
-
       ]]></BatchCLRTestLaunchCmds>
-    </PropertyGroup>   
+      <BatchCLRTestLaunchCmds><![CDATA[
+$(BatchCLRTestLaunchCmds)
+ECHO %_DebuggerFullPath% $(_CLRTestRunFile) $(InputAssemblyName) %CLRTestExecutionArguments% %Host_Args%
+%_DebuggerFullPath% $(_CLRTestRunFile) $(InputAssemblyName) %CLRTestExecutionArguments% %Host_Args%
+set CLRTestExitCode=!ERRORLEVEL!
+      ]]></BatchCLRTestLaunchCmds>
+    </PropertyGroup>
     <PropertyGroup>
       <BatchEnvironmentVariables>
 @(CLRTestBatchEnvironmentVariable -> '%(Identity)', '%0d%0a')
@@ -269,7 +292,7 @@ $(BatchCLRTestArgPrep)
       <_CLRTestExecutionScriptText>
   <![CDATA[
 @ECHO OFF
-setlocal
+setlocal ENABLEDELAYEDEXPANSION
 pushd %~dp0
 $(BatchCLRTestArgPrep)
 $(BatchCLRTestExitCodePrep)
@@ -283,6 +306,9 @@ $(BatchEnvironmentVariables)
 REM CrossGen Script (when /p:CrossGen=true)
 $(CrossgenBatchScript)
 
+REM IlasmRoundTrip Script
+$(IlasmRoundTripBatchScript)
+
 REM Precommands
 $(CLRTestBatchPreCommands)
 REM Launch
index 35c52fd744b5880d8ab36a82815dd985a5f591b1..e85d9695d81f4ff86ec4e907137d645676054832 100644 (file)
@@ -14,7 +14,6 @@
       <_OutputTypeArgument Condition="'$(OutputType)' == 'Library'">/DLL</_OutputTypeArgument>
       <_OutputTypeArgument Condition="'$(OutputType)' == 'Exe'">/EXE</_OutputTypeArgument>
       <_IlasmSwitches>/QUIET /NOLOGO</_IlasmSwitches>
-      <_IlasmSwitches Condition="'$(KeyOriginatorFile)' != ''">$(_IlasmSwitches) /KEY=$(KeyOriginatorFile)</_IlasmSwitches>
       <_IlasmSwitches Condition="'$(FoldIdenticalMethods)' == 'True'">$(_IlasmSwitches) /FOLD</_IlasmSwitches>
       <_IlasmSwitches Condition="'$(SizeOfStackReserve)' != ''">$(_IlasmSwitches) /STACK=$(SizeOfStackReserve)</_IlasmSwitches>
       <_IlasmSwitches Condition="'$(DebugType)' == 'Full'">$(_IlasmSwitches) /DEBUG</_IlasmSwitches>
index 4ddb8bc69020372a0f91627c39604ba602bb499c..eebe8b0222fdb88fa15dc5e0d6ac1dbf59554bfd 100644 (file)
@@ -14,6 +14,7 @@
     <ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
     <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+    <IlasmRoundTrip>true</IlasmRoundTrip>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
index 8427c290eca3c42d77dc2084b438bd7c395e978e..88ee59d8bdb682adfdece18be54a273d9cad1e7a 100644 (file)
@@ -13,6 +13,7 @@
     <ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
     <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+    <IlasmRoundTrip>true</IlasmRoundTrip>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
index a85f4e238c2d81240cd9815881f40c842c6c836e..5d6d3c7019628707ce906f928816ce6f3f72f8d4 100644 (file)
@@ -15,6 +15,7 @@
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
     <RestorePackages>true</RestorePackages>
     <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+    <IlasmRoundTrip>true</IlasmRoundTrip>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
index 3888e2d2061085bb902c7eef710d4a2731ae7550..eb263f8c1172dd746e6fd49aa8f3f70828dcec24 100644 (file)
@@ -15,6 +15,7 @@
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
 
     <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+    <IlasmRoundTrip>true</IlasmRoundTrip>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
index afef3a71b41ac53b7ad0b992299563a71a459df0..e6af1f79541c44014e94976328042853e97674ed 100644 (file)
@@ -15,6 +15,7 @@
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
     <RestorePackages>true</RestorePackages>
     <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+    <IlasmRoundTrip>true</IlasmRoundTrip>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
index 380db838297979bde9bbfbe2b3c3c6f11b0b8cbf..d707e581c22c19f52c983a536de77e01ff568b24 100644 (file)
@@ -15,6 +15,7 @@
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
 
     <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+    <IlasmRoundTrip>true</IlasmRoundTrip>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
index e9f416df72ca328787bcfaba070366d9feae58a2..9263d7ebd5db9f83b190556bec08348aabd5f7e9 100644 (file)
@@ -13,6 +13,7 @@
     <ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
     <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+    <IlasmRoundTrip>true</IlasmRoundTrip>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
index 54c1f9dcf1e5f19c11388b768a23709a047bb1ce..f17ec7f1c70ecc2236c5b1b506fd5121a217c626 100644 (file)
@@ -15,6 +15,7 @@
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
     <RestorePackages>true</RestorePackages>
     <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+    <IlasmRoundTrip>true</IlasmRoundTrip>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
index 85e8868c6ff217fc201518a916add4c3e723492f..d9835c33646b456b12616d8123f456ad7411b33d 100644 (file)
@@ -13,6 +13,7 @@
     <ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
     <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+    <IlasmRoundTrip>true</IlasmRoundTrip>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
index 6ed553102c798f188ede08b7bf5ed1a7ccd43ccc..93fd0cb1811bae8c9e965e0fb5e7c8ad1eb42511 100644 (file)
@@ -15,6 +15,7 @@
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
     <RestorePackages>true</RestorePackages>
     <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+    <IlasmRoundTrip>true</IlasmRoundTrip>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
index 848d302af8488c53bbb3bd6363d07f81002f2bc3..114148936522a71574ffc05376440b754be3dd56 100644 (file)
@@ -15,6 +15,7 @@
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
     <RestorePackages>true</RestorePackages>
     <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+    <IlasmRoundTrip>true</IlasmRoundTrip>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">