Re-enable multi-queue for coreclr (dotnet/coreclr#10541)

* Add multiqueue Support for coreclr.

Add multiqueue Support for coreclr.

* Upload build results to ONE linux container

Upload build results to ONE linux container

* Fix helixpublish.proj

Fix helixpublish.proj

* Adding more logs.

Adding more logs.

* Revert logging changes.

Revert logging changes.

* Use build version containing support of "+" delimiter

Use build version containing support of "+" delimiter

* Fix indention.

Fix indention.

* Update to use latest tools.

Update to use latest tools.

* Use an older tool as latest has a wired error on "Run build.sh" step.

Error details:
2017-03-28T23:30:13.2659430Z Running:
/root/coreclr/Tools/dotnetcli/dotnet /root/coreclr/Tools/run.exe
/root/coreclr/config.json build -Project=/root/coreclr/build.proj
-MsBuildLog=/flp:Verbosity=normal;LogFile=/root/coreclr/bin/Logs/System.Private.CoreLib_Checked.log
-BuildTarget -__IntermediatesDir=/root/coreclr/bin/obj/Linux.x64.Checked
-__RootBinDir=/root/coreclr/bin -BuildNugetPackage=false
-UseSharedCompilation=false -BuildArch=x64 -BuildType=Checked
-BuildOS=Linux
-OptimizationDataDir="/root/coreclr/packages/optimization.Linux-x64.IBC.CoreCLR//data/"
-EnableProfileGuidedOptimization=true -skiprestore
-OfficialBuildId=20170329-01 -- /p:ConfigurationGroup=Release
/flp:v=diag
2017-03-28T23:30:13.6855470Z Running: /root/coreclr/Tools/msbuild.sh
/nologo /verbosity:minimal /clp:Summary /maxcpucount
/l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log
/p:RestoreDuringBuild=false  /root/coreclr/build.proj
/p:__BuildType=Checked /p:__BuildArch=x64 /p:__BuildOS=Linux
/p:__RootBinDir=/root/coreclr/bin
/p:__IntermediatesDir=/root/coreclr/bin/obj/Linux.x64.Checked
/p:OfficialBuildId=20170329-01 /p:EnableProfileGuidedOptimization=true
/p:OptimizationDataDir="/root/coreclr/packages/optimization.Linux-x64.IBC.CoreCLR//data/"

/flp:Verbosity=normal;LogFile=/root/coreclr/bin/Logs/System.Private.CoreLib_Checked.log
/p:BuildNugetPackage=false /t:Build /p:UseSharedCompilation=false
/p:ConfigurationGroup=Release /flp:v=diag
2017-03-28T23:30:15.5843800Z
/root/coreclr/Tools/Microsoft.CSharp.CurrentVersion.targets(321,5):
error MSB4019: The imported project
"/root/coreclr/Tools/Microsoft.Net.Compilers/2.0.0-rc/tools/Microsoft.CSharp.Core.targets"
was not found. Confirm that the path in the <Import> declaration is
correct, and that the file exists on disk.
[/root/coreclr/src/ToolBox/SOS/NETCore/SOS.NETCore.csproj]
2017-03-28T23:30:16.6459770Z
/root/coreclr/Tools/Microsoft.CSharp.CurrentVersion.targets(321,5):
error MSB4019: The imported project
"/root/coreclr/Tools/Microsoft.Net.Compilers/2.0.0-rc/tools/Microsoft.CSharp.Core.targets"
was not found. Confirm that the path in the <Import> declaration is
correct, and that the file exists on disk.
[/root/coreclr/src/mscorlib/System.Private.CoreLib.csproj]

* New builds have errors, revert to use older builds.

New builds have errors, revert to use older builds.

Commit migrated from https://github.com/dotnet/coreclr/commit/def03cc858e599b45c165fa592a3e5b0de272e22

diff --git a/src/coreclr/publish-packages.sh b/src/coreclr/publish-packages.sh
index d5d4caa..0827a65 100755 (executable)
--- a/src/coreclr/publish-packages.sh
+++ b/src/coreclr/publish-packages.sh
@@ -10,6 +10,8 @@ usage()
     echo "   -BuildArch=\"Architecture\""
     echo "To specify the name of the container to publish into, use the following property:"
     echo "   -Container=\"container name\""
+    echo "To specify the OS you're building for, use the following property:"
+    echo "   -DistroRiD=\"RID\""       
     echo "Configuration can be Release, Checked, or Debug"
     echo "Architecture can be x64, x86, arm, or arm64"
     exit 1
@@ -18,7 +20,7 @@ usage()
 working_tree_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 $working_tree_root/run.sh publish-packages -BuildOS $*
-$working_tree_root/run.sh publish-packages -BuildOS -distroRid -PublishTestNativeBins $*
+$working_tree_root/run.sh publish-packages -BuildOS -PublishTestNativeBins $*
 if [ $? -ne 0 ]
 then
     echo "ERROR: An error occurred while publishing packages; see $working_tree_root/publish-packages.log for more details. There may have been networking problems, so please try again in a few minutes."

diff --git a/src/coreclr/tests/helixpublish.proj b/src/coreclr/tests/helixpublish.proj
index 699e220..fd9713e 100644 (file)
--- a/src/coreclr/tests/helixpublish.proj
+++ b/src/coreclr/tests/helixpublish.proj
@@ -1,73 +1,97 @@
-<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  
+<Project ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="..\dir.props" />
+  <!-- CoreClr-Specific Helix test submission project. -->
+
   <PropertyGroup>
-    <__BuildType>$(ConfigurationGroup)</__BuildType>
+    <!-- Workaround for dealing with building on Linux/OSX.  
+         It doesn't seem possible to pass a connection string on the command line, as it contains ';' characters. 
+         Checking for the existence of a connection string will happen inside CloudTest.Helix.targets -->
+    <CloudDropConnectionString    Condition="'$(CloudDropConnectionString)'==''"   >DefaultEndpointsProtocol=https;AccountName=$(CloudDropAccountName);AccountKey=$(CloudDropAccessToken);EndpointSuffix=core.windows.net</CloudDropConnectionString>
+    <CloudResultsConnectionString Condition="'$(CloudResultsConnectionString)'==''">DefaultEndpointsProtocol=https;AccountName=$(CloudResultsAccountName);AccountKey=$(CloudResultsAccessToken);EndpointSuffix=core.windows.net</CloudResultsConnectionString>    
+
+    <!-- Finally, these archives represent the zips of tests that are OSPlatform specific 
+         This is used by CloudTest.Helix.Targets to generate relative blob paths for archives. -->
+    <ArchivesRoot>$(TestWorkingDir)/archive/</ArchivesRoot>
+    <TestArchivesRoot>$(ArchivesRoot)tests</TestArchivesRoot> 
+
+    <RunnerScript Condition="'$(RunnerScript)' == ''">$(TestRunnerScript)</RunnerScript>
+    <SupplementalPayloadDir Condition="'$(SupplementalPayloadDir)' == ''">$(TestWorkingDir)SupplementalPayload/</SupplementalPayloadDir>
+
+    <!-- The old CloudTest.targets forced a single timeout value onto every test -->
+    <!-- Possible TODO: With the CloudTest.Helix.targets refactor, we have the ability to specify variable timeout-->
+    <TimeoutInSeconds Condition="'$(TimeoutInSeconds)' == ''">1000</TimeoutInSeconds>
+
+    <!-- Set required properties for Helix Submission -->
+    <IsOfficial Condition="'$(OfficialBuildId)'!=''">true</IsOfficial>
+    <CurrentDate>$([System.DateTime]::Now.ToString(yyyyMMdd.mmss))</CurrentDate>
+
+    <HelixJobType Condition="'$(HelixJobType)'==''">test/functional/cli/</HelixJobType>
+
+    <HelixSource Condition="'$(HelixSource)'=='' And '$(IsOfficial)'!='' And '$(TestProduct)'!='' And '$(Branch)'!=''">official/$(TestProduct)/$(Branch)/</HelixSource>
+    <HelixSource Condition="'$(HelixSource)'=='' And '$(IsOfficial)'=='' And '$(TestProduct)'!='' And '$(Branch)'!=''">pr/$(TestProduct)/$(Branch)/</HelixSource>
+    <HelixSource Condition="'$(HelixSource)'==''">pr/unknown/</HelixSource>
+
+    <!-- Local path for storing submission-related files -->
+    <HelixLogFolder>$(ArchivesRoot)</HelixLogFolder>
+
+    <!-- Properties used for submission by CloudTest.Helix.Targets-->
+    <BuildMoniker>$(CurrentDate)</BuildMoniker>
+    <BuildMoniker Condition="'$(IsOfficial)'=='true'">$(OfficialBuildId)</BuildMoniker>
+    <HelixArchLabel>$(Platform)</HelixArchLabel>
+    <HelixConfigLabel>$(BuildType)</HelixConfigLabel>
   </PropertyGroup>
 
-  <Import Project="..\dir.props" />
-  <Import Project="$(ToolsDir)CloudTest.targets" Condition="Exists('$(ToolsDir)CloudTest.targets')" />
-
-  <!-- Define test payload & Correlation (Core_Root) payload -->
-  <ItemGroup>
-    <TestList Include="$(TestWorkingDir)\archive\tests\*" ></TestList>
-    <CoreRootUri Include="$(TestWorkingDir)\archive\Core_Root*\*.zip" ></CoreRootUri>
-    <DummyPackages Include="$(TestWorkingDir)\archive\packages\*" ></DummyPackages>
-    <ForUpload Include="@(TestList)" ></ForUpload>
-    <ForUpload Include="@(CoreRootUri)" ></ForUpload>
-  </ItemGroup>
-
-  <!-- Define name & location of test JSON blob -->
-  <PropertyGroup>
-    <SkipArchive>true</SkipArchive>
-    <PayloadTestListFilename>Tests.$(ConfigurationGroup).json</PayloadTestListFilename>
-    <PayloadTestListFile>$(TestWorkingDir)$(PayloadTestListFilename)</PayloadTestListFile>
+  <!-- Set Helix environment vars based on target platform -->
+  <!-- This is only used in the case where property 'UseScriptRunner' is true.-->
+  <PropertyGroup Condition="'$(TargetsWindows)' == 'true'">
+    <HelixPythonPath>%HELIX_PYTHONPATH%</HelixPythonPath>
+    <RunnerScript>%HELIX_CORRELATION_PAYLOAD%\RunnerScripts\scriptrunner\scriptrunner.py</RunnerScript>
   </PropertyGroup>
 
-  <Target Name="CreateTestListJson"
-          DependsOnTargets="CreateAzureStorage">
+  <PropertyGroup Condition="'$(TargetsWindows)' != 'true'">
+    <HelixPythonPath>$HELIX_PYTHONPATH</HelixPythonPath>
+    <RunnerScript>$HELIX_CORRELATION_PAYLOAD/RunnerScripts/scriptrunner/scriptrunner.py</RunnerScript>
+  </PropertyGroup>
 
+  <Target Name="CoreClrPreCloudBuild" >
+    <!-- Put anything you want to do before submitting the general purpose helix job here.
+         Helix requires properly formatted work items and accessible zip files / others to upload, which we'll construct here. -->
+    
+    <!-- Gather the test archives for this build 
+         If you change binplacing behavior and official runs break, 
+         this is a good place to check.-->
     <ItemGroup>
-      <SupplementalPayload Include="@(DummyPackages)" >
-        <RelativeBlobPath>$(Platform)$(ConfigurationGroup)/Packages.zip</RelativeBlobPath>
-      </SupplementalPayload>
-      <CorrelationPayloadUri Include="@(SupplementalPayload->'$(DropUri)%(RelativeBlobPath)$(DropUriReadOnlyToken)')" />
+      <HelixWorkItem Include="$(TestArchivesRoot)/*.zip" /> 
     </ItemGroup>
+    <Message Text="Full test archive collection : @(HelixWorkItem)" Importance="Low" />
 
-    <PropertyGroup>
-      <!-- flatten it into a property as msbuild chokes on @(CorrelationPayloadUri) -->
-      <CorrelationPayloadUris>@(CorrelationPayloadUri)</CorrelationPayloadUris>
-      <!-- Define Correlation Payload as a property -->
-      <CoreRootUris>$(DropUri)$(Platform)$(ConfigurationGroup)/Tests/$([System.String]::Copy('%(RecursiveDir)').Replace('\', '/'))%(CoreRootUri.Filename)%(CoreRootUri.Extension)$(DropUriReadOnlyToken)</CoreRootUris>
-      <CorrelationPayloadProperty>$(CorrelationPayloadUris);$(CoreRootUris)</CorrelationPayloadProperty>
-    </PropertyGroup>
+    <!-- Verify the test archives exist.  
+         NOTE: It may be better to make this an error, but in the past we chose not to as this causes a build break when triggered, versus just no tests run. -->
+    <Warning Condition="'@(HelixWorkItem->Count())' == '0'" Text="Didn't find any test archives in supplied folders!!! Review logs to see where test archive(s) were placed and fix." />
 
-    <ItemGroup>
-      <TestList>
+    <ItemGroup>      
+      <HelixCorrelationPayloadFile Include="$(ArchivesRoot)Core_Root*/*.zip">
+        <RelativeBlobPath>$(Platform)$(ConfigurationGroup)/%(Identity)</RelativeBlobPath>
+      </HelixCorrelationPayloadFile>
+      <HelixWorkItem>
         <Command Condition="'$(TargetsWindows)' == 'true'">$(HelixPythonPath) $(RunnerScript) --script %HELIX_WORKITEM_PAYLOAD%\runtests.cmd</Command>
         <Command Condition="'$(TargetsWindows)' != 'true'">chmod +x $HELIX_WORKITEM_PAYLOAD/runtests.sh &amp;&amp; $(HelixPythonPath) $(RunnerScript) --script $HELIX_WORKITEM_PAYLOAD/runtests.sh</Command>
-        <CorrelationPayloadUris>[$(CorrelationPayloadProperty)]</CorrelationPayloadUris>
-        <PayloadUri>$(DropUri)$(Platform)$(ConfigurationGroup)/Tests/$([System.String]::Copy('%(RecursiveDir)').Replace('\', '/'))%(Filename)%(Extension)$(DropUriReadOnlyToken)</PayloadUri>
+        <PayloadFile>%(Identity)</PayloadFile>
         <WorkItemId>%(Filename)</WorkItemId>
         <TimeoutInSeconds>$(TimeoutInSeconds)</TimeoutInSeconds>
-      </TestList>
-    </ItemGroup>
-    <WriteItemsToJson JsonFileName="$(PayloadTestListFile)" Items="@(TestList)" />
-    <!-- add test lists to the list of items for upload -->
-    <ItemGroup>
-      <ForUpload Include="$(PayloadTestListFile)">
-        <RelativeBlobPath>$(PayloadTestListFilename)</RelativeBlobPath>
-      </ForUpload>
-    </ItemGroup>
-    <!-- for completion event -->
-    <ItemGroup>
-      <TestListFile Include="$(PayloadTestListFile)">
-        <BuildCompleteJson>$(TestWorkingDir)$(OSPlatformConfig)/FuncBuildComplete.json</BuildCompleteJson>
-        <OfficialBuildJson>$(TestWorkingDir)$(OSPlatformConfig)/FuncOfficialBuild.json</OfficialBuildJson>
-        <HelixJobUploadCompletePath>$(TestWorkingDir)$(OSPlatformConfig)/helixjobuploadcomplete.sem</HelixJobUploadCompletePath>
-      </TestListFile>
+        <RelativeBlobPath>$(Platform)-$(ConfigurationGroup)/Tests/$([System.String]::Copy('%(RecursiveDir)').Replace('\', '/'))%(Filename)%(Extension)</RelativeBlobPath>
+      </HelixWorkItem>      
     </ItemGroup>
   </Target>
 
-  <Target Name="Build" />
+  <!-- Import this at the end so that it can compose properties as needed -->
+  <Import Project="$(ToolsDir)CloudTest.Helix.targets" />
 
+  <!-- main Entrypoint -->
+  <Target Name="Build">
+    <Message Text="CoreClr-specific Helix test upload project, beginning submission to Helix" />
+    <Message Condition="'$(EnableCloudTest)' == 'false'" Text="... skipping upload / submission due to property 'EnableCloudTest' being set to 'false'" />
+    <CallTarget Condition="'$(EnableCloudTest)' != 'false'" Targets="CoreClrPreCloudBuild;HelixCloudBuild" />
+  </Target>
+  
 </Project>
\ No newline at end of file

diff --git a/src/coreclr/tests/src/Common/build_against_pkg_dependencies/project.json b/src/coreclr/tests/src/Common/build_against_pkg_dependencies/project.json
index 9e5e124..8b5e819 100644 (file)
--- a/src/coreclr/tests/src/Common/build_against_pkg_dependencies/project.json
+++ b/src/coreclr/tests/src/Common/build_against_pkg_dependencies/project.json
@@ -26,6 +26,7 @@
     "rhel.7-x64": {},
     "debian.8-x64": {},
     "fedora.23-x64": {},
-    "opensuse.42.1-x64": {}
+    "opensuse.42.1-x64": {},
+    "linux-x64": {}
   }
 }

diff --git a/src/coreclr/tests/src/Common/test_dependencies/project.json b/src/coreclr/tests/src/Common/test_dependencies/project.json
index ca2915e..2b7c9ea 100644 (file)
--- a/src/coreclr/tests/src/Common/test_dependencies/project.json
+++ b/src/coreclr/tests/src/Common/test_dependencies/project.json
@@ -24,6 +24,7 @@
     "centos.7-x64": {},
     "rhel.7-x64": {},
     "debian.8-x64": {},
-    "fedora.23-x64": {}
+    "fedora.23-x64": {},
+    "linux-x64": {}
   }
 }