tests/runtest.proj

   1 <?xml version="1.0" encoding="utf-8"?>
   2 <Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   3   <Import Project="src\dir.props" />
   4   <Import Project="$(ToolsDir)Build.Post.targets" Condition="Exists('$(ToolsDir)Build.Post.targets') AND '$(BuildWrappers)' == 'true'" />
   5   <Import Project="helixperftasks.targets" Condition="'$(Performance)'=='true'"/>
   6   <PropertyGroup>
   7     <XunitTestBinBase Condition="'$(XunitTestBinBase)'==''" >$(BaseOutputPathWithConfig)</XunitTestBinBase>
   8     <XunitWrapperGeneratedCSDirBase>$(XunitTestBinBase)\TestWrappers\</XunitWrapperGeneratedCSDirBase>
   9     <XunitWrapperOutputIntermediatedDirBase>$(XunitTestBinBase)\Tests\</XunitWrapperOutputIntermediatedDirBase>
  10     <MSBuildEnableAllPropertyFunctions>1</MSBuildEnableAllPropertyFunctions>
  11   </PropertyGroup>
  12
  13   <ItemGroup>
  14     <DisabledTestDir Include="Common" />
  15     <DisabledTestDir Include="tracing" />
  16     <_SkipTestDir Include="@(DisabledTestDir)" />
  17   </ItemGroup>
  18
  19   <Target Name="Rebuild" />
  20
  21   <Target Name="FindCmdDirectories" DependsOnTargets="GetListOfTestCmds">
  22
  23     <Error Condition="!Exists('$(XunitTestBinBase)')"
  24            Text="$(XunitTestBinBase) does not exist. Please run buildtest.cmd from the (repo root)\tests at least once to get the tests built." />
  25
  26     <ItemGroup>
  27
  28       <AllTestDirsNonCanonicalPaths Include="$([System.IO.Directory]::GetDirectories(`$(XunitTestBinBase)`))" />
  29       <AllTestDirsPaths Include="@(AllTestDirsNonCanonicalPaths)" />
  30       <AllTestDirsPaths Include="@(AllTestDirsNonCanonicalPaths)" >
  31         <Path>$([System.IO.Path]::GetFullPath(%(Identity)))</Path>
  32       </AllTestDirsPaths>
  33       <NonExcludedTestDirectories Include="@(AllTestDirsPaths -> '%(Path)')" Exclude="@(_SkipTestDir -> '$(XunitTestBinBase)%(Identity)')" />
  34       <TopLevelDirectories Include="@(NonExcludedTestDirectories)" />
  35       <SecondLevel Include="$([System.IO.Directory]::GetDirectories(%(TopLevelDirectories.Identity)))" />
  36       <SecondLevelDirectories Include="@(SecondLevel)">
  37         <Path>$([System.IO.Path]::GetFullPath(%(AllRunnableTestPaths.Identity)))</Path>
  38       </SecondLevelDirectories>
  39       <TestDirectoriesWithDup Include="@(SecondLevelDirectories -> '%(Identity)')" Condition="$([System.String]::new('%(Path)').StartsWith('%(Identity)'))" />
  40
  41     </ItemGroup>
  42
  43     <RemoveDuplicates Inputs="@(TestDirectoriesWithDup)">
  44       <Output
  45           TaskParameter="Filtered"
  46           ItemName="TestDirectories"/>
  47     </RemoveDuplicates>
  48
  49   </Target>
  50
  51   <Import Project="$(__Exclude)" Condition="'$(__Exclude)' != '' AND '$(XunitTestBinBase)' != ''" />
  52   <PropertyGroup>
  53     <HaveExcludes>False</HaveExcludes>
  54     <HaveExcludes Condition="'$(__Exclude)' != ''">True</HaveExcludes>
  55   </PropertyGroup>
  56
  57   <Target Name="CreateXunitWrapper" DependsOnTargets="CreateXunitFacts">
  58
  59     <PropertyGroup>
  60       <_XunitWrapperGen >
  61         <![CDATA[
  62
  63 $(_XunitProlog)
  64 @(AllXUnitFacts)
  65 $(_XunitEpilog)
  66
  67 ]]>
  68       </_XunitWrapperGen>
  69       <XunitWrapperGenCsProj>
  70         <![CDATA[
  71 <?xml version="1.0" encoding="utf-8"?>
  72 <Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
  73
  74   <Import Project="$(SourceDir)dir.props" />
  75   <PropertyGroup>
  76     <Configuration Condition=" '%24(Configuration)' == '' ">Debug</Configuration>
  77     <Platform Condition=" '%24(Platform)' == '' ">AnyCPU</Platform>
  78     <AssemblyName>$(XunitWrapper)</AssemblyName>
  79     <SchemaVersion>2.0</SchemaVersion>
  80     <ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
  81     <OutputType>Library</OutputType>
  82     <TargetFrameworkIdentifier Condition ="('$(BuildTestsAgainstPackages)' != 'true') And ('$(BuildOS)' == 'Windows_NT')">.NETFramework</TargetFrameworkIdentifier>
  83     <TargetFrameworkVersion Condition ="('$(BuildTestsAgainstPackages)' != 'true') And ('$(BuildOS)' == 'Windows_NT')">v4.5</TargetFrameworkVersion>
  84     <NugetTargetMonikerShort Condition ="'$(BuildTestsAgainstPackages)' != 'true'">net45</NugetTargetMonikerShort>
  85     <IsXunitWrapperProject>true</IsXunitWrapperProject>
  86     <SkipSigning>true</SkipSigning>
  87     <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB}%3B{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
  88     <SolutionDir Condition="%24(SolutionDir) == '' Or %24(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
  89     <CLRTestKind>BuildOnly</CLRTestKind>
  90     <IsTestProject>true</IsTestProject>
  91   </PropertyGroup>
  92   <!-- Default configurations to help VS understand the configurations -->
  93   <PropertyGroup Condition=" '%24(Configuration)|%24(Platform)' == 'Debug|AnyCPU' ">
  94   </PropertyGroup>
  95   <PropertyGroup Condition=" '%24(Configuration)|%24(Platform)' == 'Release|AnyCPU' ">
  96   </PropertyGroup>
  97   <ItemGroup>
  98     <Compile Include="$(XunitWrapper).cs" />
  99   </ItemGroup>
 100   <ItemGroup>
 101     <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
 102   </ItemGroup>
 103    <ItemGroup>
 104     <ProjectReference Include="$(SourceDir)Common\Desktop.Coreclr.TestWrapper\Desktop.Coreclr.TestWrapper.csproj" Condition="('$(BuildTestsAgainstPackages)' != 'true') And ('$(BuildOS)' == 'Windows_NT')">
 105       <Project>{8ffe99c0-22f8-4462-b839-970eac1b3472}</Project>
 106       <Name>coreclr</Name>
 107     </ProjectReference>
 108     <ProjectReference Include="$(SourceDir)Common\Coreclr.TestWrapper\Coreclr.TestWrapper.csproj" Condition="('$(BuildTestsAgainstPackages)' == 'true') Or ('$(BuildOS)' != 'Windows_NT')">
 109       <Project>{8ffe99c0-22f8-4462-b839-970eac1b3472}</Project>
 110       <Name>coreclr</Name>
 111     </ProjectReference>
 112   </ItemGroup>
 113
 114   <ItemGroup>
 115     <Reference Include="System" />
 116     <Reference Include="System.Runtime" />
 117     <Reference Include="mscorlib" />
 118   </ItemGroup>
 119   <Import Project="$(SourceDir)dir.targets" />
 120   <PropertyGroup>
 121     <OutDir>$(XunitTestBinBase)\$(CategoryWithSlash)\</OutDir>
 122     <ProjectAssetsFile Condition="'$(BuildTestsAgainstPackages)' != 'true'">%24(TestWrappersPackagesConfigFileDirectory)obj/project.assets.json</ProjectAssetsFile>
 123   </PropertyGroup>
 124 </Project>
 125         ]]>
 126       </XunitWrapperGenCsProj>
 127
 128     </PropertyGroup>
 129
 130     <!-- <Exec Command="md" -->
 131     <MakeDir  Directories="$(XunitWrapperGeneratedCSDirBase)$(Category)"/>
 132
 133     <!-- Write the file -->
 134     <WriteLinesToFile
 135       File="$(XunitWrapperSrcDir)\$(XunitWrapper).cs"
 136       Lines="$(_XunitWrapperGen)"
 137       Overwrite="true" />
 138
 139     <!-- Write the file -->
 140     <WriteLinesToFile
 141       File="$(XunitWrapperSrcDir)\$(XunitWrapper).csproj"
 142       Lines="$(XunitWrapperGenCsProj)"
 143       Overwrite="true" />
 144   </Target>
 145
 146   <Target Name="BuildXunitWrapper">
 147     <MSBuild Projects="$(XunitWrapperSrcDir)\$(XunitWrapper).csproj"/>
 148   </Target>
 149
 150   <Target Name="CreateXunitFacts">
 151     <!-- NOTE! semicolons must be escaped with %3B boooo -->
 152
 153     <PropertyGroup>
 154       <_CMDDIR_Parent>$([System.IO.Path]::GetDirectoryName($(_CMDDIR)))</_CMDDIR_Parent>
 155       <_CMDDIR_Grandparent>$([System.IO.Path]::GetDirectoryName($(_CMDDIR_Parent)))</_CMDDIR_Grandparent>
 156       <CategoryWithSlash Condition="'$(RunningOnUnix)' != 'true'" >$([System.String]::Copy('$(_CMDDIR)').Replace("$(_CMDDIR_Grandparent)\",""))</CategoryWithSlash>
 157       <CategoryWithSlash Condition="'$(RunningOnUnix)' == 'true'" >$([System.String]::Copy('$(_CMDDIR)').Replace("$(_CMDDIR_Grandparent)/",""))</CategoryWithSlash>
 158       <Category Condition="'$(RunningOnUnix)' != 'true'" >$([System.String]::Copy('$(CategoryWithSlash)').Replace('\','.'))</Category>
 159       <Category Condition="'$(RunningOnUnix)' == 'true'" >$([System.String]::Copy('$(CategoryWithSlash)').Replace('/','.'))</Category>
 160       <XunitWrapper>$(Category).XUnitWrapper</XunitWrapper>
 161       <XunitWrapperSrcDir>$(XunitWrapperGeneratedCSDirBase)$(Category)</XunitWrapperSrcDir>
 162       <XunitWrapperOutputDir>$(XunitWrapperOutputIntermediatedDirBase)$(Category)</XunitWrapperOutputDir>
 163     </PropertyGroup>
 164     <PropertyGroup>
 165       <_XunitProlog Condition=" '$(_XunitProlog)'=='' ">
 166         <![CDATA[
 167 using Xunit%3B
 168 using System%3B
 169 using System.Collections.Generic%3B
 170 using System.Diagnostics%3B
 171 using System.Reflection%3B
 172 using CoreclrTestLib%3B
 173
 174 namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").Replace("-","_"))
 175 {
 176         internal class _Global
 177         {
 178             internal static bool runningInWindows%3B
 179             internal static string reportBase%3B
 180             internal static string testBinaryBase%3B
 181             internal static string coreRoot%3B
 182
 183             static _Global()
 184             {
 185                 reportBase = System.Environment.GetEnvironmentVariable(%22XunitTestReportDirBase%22)%3B
 186                 testBinaryBase = System.IO.Path.GetDirectoryName(new Uri(Assembly.GetExecutingAssembly().CodeBase).LocalPath)%3B
 187                 coreRoot = System.IO.Path.GetFullPath(System.Environment.GetEnvironmentVariable(%22CORE_ROOT%22))%3B
 188
 189                 if (String.IsNullOrEmpty(reportBase)) {
 190                     reportBase = System.IO.Path.Combine(testBinaryBase, "Reports")%3B
 191                 }
 192                 else
 193                 {
 194                     reportBase = System.IO.Path.GetFullPath(reportBase)%3B
 195                 }
 196
 197                 if (String.IsNullOrEmpty(coreRoot)) {
 198                     throw new ArgumentException("Environment variable CORE_ROOT is not set")%3B
 199                 }
 200
 201                 string operatingSystem = System.Environment.GetEnvironmentVariable("OS")%3B
 202                 runningInWindows = (operatingSystem != null && operatingSystem.StartsWith("Windows"))%3B
 203             }
 204         }
 205
 206 ]]>
 207       </_XunitProlog>
 208
 209       <_XunitEpilog Condition=" '$(_XunitEpilog)'=='' ">
 210       <![CDATA[
 211 }
 212
 213 ]]>
 214       </_XunitEpilog>
 215     </PropertyGroup>
 216
 217     <ItemGroup>
 218       <CanonicalExcludeList Include="%(ExcludeList.FullPath)" Condition="$(HaveExcludes)"/>
 219     </ItemGroup>
 220
 221     <PropertyGroup>
 222       <TestExecutableReplacement Condition="'$(TargetsWindows)' != 'true' ">testExecutable = testExecutable.Replace("\\", "/")%3B</TestExecutableReplacement>
 223     </PropertyGroup>
 224
 225     <ItemGroup>
 226       <AllCMDsPresent Include="$(_CMDDIR)\**\*.cmd" Condition="'$(RunningOnUnix)' != 'true'" />
 227       <AllCMDsPresent Include="$(_CMDDIR)\**\*.sh" Condition="'$(RunningOnUnix)' == 'true'" />
 228       <AllCMDExcludeFilter Include="@(CanonicalExcludeList)" Condition="$(HaveExcludes)"/>
 229       <AllCMDs Include="@(AllCMDsPresent)" Exclude="@(AllCMDExcludeFilter)"/>
 230
 231       <AllCommands Include="@(AllCMDs)" >
 232         <_FactName Condition="'$(RunningOnUnix)' != 'true'" >$([System.String]::Copy('%(AllCMDs.FullPath)').Replace("$(_CMDDIR)",'').Replace(".","_").Replace("\","_").Replace("-","_"))</_FactName>
 233         <_ClassName Condition="'$(RunningOnUnix)' != 'true'" >$([System.String]::Copy('%(AllCMDs.FullPath)').Replace("$(_CMDDIR)",'').Replace("cmd","").Replace(".","_").Replace("\","_").Replace("-","_"))</_ClassName>
 234         <_FactName Condition="'$(RunningOnUnix)' == 'true'" >$([System.String]::Copy('%(AllCMDs.FullPath)').Replace("$(_CMDDIR)",'').Replace(".","_").Replace("/","_").Replace("-","_"))</_FactName>
 235         <_ClassName Condition="'$(RunningOnUnix)' == 'true'" >$([System.String]::Copy('%(AllCMDs.FullPath)').Replace("$(_CMDDIR)",'').Replace("sh","").Replace(".","_").Replace("/","_").Replace("-","_"))</_ClassName>
 236
 237         <_XunitFact >
 238           <![CDATA[
 239
 240         public class %(AllCommands._ClassName)
 241         {
 242             [Fact]
 243             public void %(AllCommands._FactName)()
 244             {
 245                 int ret = -100%3B
 246                 string outputFile = null%3B
 247                 string errorFile = null%3B
 248                 string testExecutable = null%3B
 249                 Exception infraEx = null%3B
 250
 251                 try
 252                 {
 253                   CoreclrTestWrapperLib wrapper = new CoreclrTestWrapperLib()%3B
 254                   string testSubfolder = @"\$(Category)\$([System.String]::Copy('%(AllCMDs.RelativeDir)').Replace("$(_CMDDIR)\",''))"%3B
 255                   outputFile = System.IO.Path.GetFullPath(_Global.reportBase + testSubfolder + @"%(AllCMDs.FileName).output.txt")%3B
 256                   errorFile = System.IO.Path.GetFullPath(_Global.reportBase + testSubfolder + @"%(AllCMDs.FileName).error.txt")%3B
 257                   testExecutable = System.IO.Path.GetFullPath(_Global.testBinaryBase + @"$([System.String]::Copy('%(AllCMDs.FullPath)').Replace("$(_CMDDIR)",''))")%3B
 258                   $(TestExecutableReplacement)
 259
 260                   if (!_Global.runningInWindows) {
 261                       testExecutable = testExecutable.Replace(".cmd", ".sh")%3B
 262                   }
 263
 264                   System.IO.Directory.CreateDirectory(_Global.reportBase + testSubfolder)%3B
 265
 266                   ret = wrapper.RunTest(testExecutable, outputFile, errorFile)%3B
 267                 }
 268                 catch (Exception ex)
 269                 {
 270                     infraEx = ex%3B
 271                 }
 272
 273                 if (ret != CoreclrTestWrapperLib.EXIT_SUCCESS_CODE)
 274                 {
 275                     string sErrorText = null%3B
 276                     try
 277                     {
 278                         sErrorText = System.IO.File.ReadAllText(errorFile)%3B
 279                     }
 280                     catch(Exception ex)
 281                     {
 282                       sErrorText = "Unable to read error file: " + errorFile%3B
 283                     }
 284
 285                     string outputText = null%3B
 286                     try
 287                     {
 288                         System.IO.StreamReader outputReader = new System.IO.StreamReader(outputFile)%3B
 289                         outputText = outputReader.ReadToEnd()%3B
 290                         outputReader.Close()%3B
 291                     }
 292                     catch(Exception ex)
 293                     {
 294                         outputText = "Unable to read output file: " + outputFile%3B
 295                     }
 296
 297                     string msg = infraEx != null ? "Test Infrastructure Failure: " + infraEx.Message
 298                                                  : sErrorText + "\n\n" +
 299                                                    "Return code:      " + ret + "\n" +
 300                                                    "Raw output file:      " + outputFile + "\n" +
 301                                                    "Raw output:\n" + outputText + "\n" +
 302                                                    "To run the test:\n" +
 303                                                    "> set CORE_ROOT=" + _Global.coreRoot + "\n" +
 304                                                    "> " + testExecutable + "\n"%3B
 305
 306                     Assert.True(ret == CoreclrTestWrapperLib.EXIT_SUCCESS_CODE, msg)%3B
 307                 }
 308             }
 309        }
 310
 311         ]]>
 312         </_XunitFact>
 313       </AllCommands>
 314       <AllXUnitFacts Include= "%(AllCommands._XunitFact)" />
 315     </ItemGroup>
 316
 317   </Target>
 318
 319   <Target Name="CreateAllWrappers" DependsOnTargets="GetListOfTestCmds;FindCmdDirectories">
 320     <MSBuild Projects="$(MSBuildProjectFile)" Targets="CreateXunitWrapper;BuildXunitWrapper" Properties="_CMDDIR=%(TestDirectories.Identity)" />
 321   </Target>
 322
 323   <Target Name="GetListOfTestCmds">
 324     <ItemGroup>
 325       <AllRunnableTestPaths Include="$(XunitTestBinBase)\**\*.cmd" />
 326     </ItemGroup>
 327   </Target>
 328
 329   <!-- If we want to overwrite the desired CoreCLR package version, we need to get the new version from the generated props file in bin/obj -->
 330   <Import Condition="'$(OverwriteCoreClrPackageVersion)' == 'true'" Project="$(BuildVersionFile)" />
 331
 332   <PropertyGroup Condition="'$(OverwriteCoreClrPackageVersion)' == 'true'">
 333     <VersionToRestore Condition="'$(StableVersion)' != ''">$(StableVersion)</VersionToRestore>
 334     <VersionToRestore Condition="'$(VersionToRestore)' == ''">$(PackageVersion)-$(PreReleaseLabel)-$(BuildNumberMajor)-$(BuildNumberMinor)</VersionToRestore>
 335     <DesiredPackageVersionArg>$(VersionToRestore)</DesiredPackageVersionArg>
 336   </PropertyGroup>
 337
 338   <Import Project="tests.targets" />
 339   <Import Project="publishdependency.targets" />
 340
 341   <PropertyGroup>
 342     <DesiredPackageVersionArg Condition="'$(DesiredPackageVersionArg)' == ''">$(CoreClrPackageVersion)</DesiredPackageVersionArg>
 343   </PropertyGroup>
 344
 345   <Target Name="CreateTestOverlay">
 346     <MSBuild Projects="$(MSBuildProjectFile)"
 347              Targets="CopyDependecyToCoreRoot"
 348              Properties="Language=C#;TargetRid=$(TargetRid)" />
 349   </Target>
 350
 351   <Target Name="BinPlaceRef">
 352     <!-- Copy mscorlib.dll from TargetingPack to bin/Product/ref, if we're building against packages -->
 353     <MSBuild Projects="$(MSBuildProjectFile)"
 354              Targets="CopyDependencyToRef"
 355              Properties="Language=C#;TargetRid=$(TargetRid)"
 356              Condition=" '$(BuildTestsAgainstPackages)'=='true' " />
 357   </Target>
 358
 359   <Target Name="BinPlaceProduct">
 360     <!-- Copy test dependencies to bin/Product, if we're building against packages -->
 361     <MSBuild Projects="$(MSBuildProjectFile)"
 362              Targets="CopyDependencyToProduct"
 363              Properties="Language=C#;TargetRid=$(TargetRid)"
 364              Condition=" '$(BuildTestsAgainstPackages)'=='true' " />
 365   </Target>
 366
 367   <Target Name="RunPerfTests" Condition="'$(Performance)'=='true'">
 368     <Message Text="Executing steps for perf tests" Importance="High"/>
 369
 370     <!-- generate project.json for runtime dependency -->
 371     <MSBuild Projects="$(MSBuildProjectFile)"
 372              Targets="CreateTestRuntimeJsonFile"/>
 373
 374     <!-- generate project.lock.json file corresponding to above json file -->
 375     <MSBuild Projects="src\Common\test_dependencies\test_dependencies.csproj"/>
 376
 377     <!-- Package each perf test, upload it and trigger event to execute the tests -->
 378     <MSBuild Projects="$(MSBuildProjectFile)" Targets="UploadPerfAssemblies" />
 379   </Target>
 380
 381
 382   <Target Name="Build">
 383
 384     <!-- generate project.lock.json file corresponding to above json file -->
 385     <MSBuild Projects="src\Common\test_dependencies\test_dependencies.csproj"
 386              Condition=" '$(BuildWrappers)'=='true' " />
 387
 388     <!-- Default for building -->
 389     <MSBuild Projects="$(MSBuildProjectFile)"
 390              Targets="CreateAllWrappers"
 391              Properties="_CMDDIR=%(TestDirectories.Identity)"
 392              Condition=" '$(BuildWrappers)'=='true' " />
 393
 394     <!-- Execution -->
 395
 396     <MSBuild Projects="$(MSBuildProjectFile)"
 397              Targets="CreateTestOverlay"
 398              Condition=" '$(GenerateRuntimeLayout)'=='true' "/>
 399
 400     <MSBuild Projects="$(MSBuildProjectFile)" Targets="RunTests"
 401              Condition=" '$(RunTests)'=='true' "/>
 402   </Target>
 403
 404   <Target Name="Clean">
 405     <RemoveDir Condition=" '$(BuildWrappers)'=='true'" Directories="$(XunitWrapperGeneratedCSDirBase);$(XunitWrapperOutputIntermediatedDirBase)" ContinueOnError="WarnAndContinue" />
 406   </Target>
 407 </Project>