[netcore] Improve xtest command (mono/mono#13703)

The python script now also extracts downloaded zips and `make xtest-%` is now able to consume these artifacts.
E.g.
```
make xtest-System.Collections.Tests COREFX_ROOT=/prj/corefx-repo
```
We still need COREFX_ROOT because of two missing files:
`CoreFx.Private.TestUtilities.dll` and `xunit.console.deps.json`
maybe I should just add them to our repo?

Also this PR fixes `RemoteExecutorConsoleApp` problems for `make xtest-%`

Commit migrated from https://github.com/mono/mono/commit/2522547cd5ba592e21fdba2e94a9feeb20037daf

diff --git a/src/mono/netcore/.gitignore b/src/mono/netcore/.gitignore
index 8130516..ff86037 100644 (file)
--- a/src/mono/netcore/.gitignore
+++ b/src/mono/netcore/.gitignore
@@ -6,4 +6,7 @@ sdk
 dotnet*
 netcoreapp2.0/
 netcoreapp3.0/
-packs/
\ No newline at end of file
+packs/
+.stamp-dl-test-assets
+assets/
+corefx-test-assets.xml

diff --git a/src/mono/netcore/Makefile.am b/src/mono/netcore/Makefile.am
index 76acc47..72c6ed4 100644 (file)
--- a/src/mono/netcore/Makefile.am
+++ b/src/mono/netcore/Makefile.am
@@ -88,14 +88,48 @@ check: $(addprefix run-, $(TEST_SUITES))
 clean:
        rm -rf sdk shared host dotnet tests LICENSE.txt ThirdPartyNotices.txt $(NETCORESDK_FILE)
 
-xtest-%: prepare check-env
-       COMPlus_DebugWriteToStdErr=1 ./dotnet --fx-version "$(VERSION)" \
-               $(COREFX_BINDIR)/testhost/netcoreapp-OSX-Debug-x64/shared/Microsoft.NETCore.App/9.9.9/xunit.console.dll \
-               $(COREFX_BINDIR)/$*/$($*_PROFILE)/$*.dll -notrait category=outerloop -notrait category=nonosxtests -notrait category=failing \
-               -notrait category=nonnetcoreapptests -noappdomain -noshadow -parallel all -nunit TestResult-$*-netcore-xunit.xml \
-               $(shell grep -v '^#\|^$$' excludes-$*.rsp) $(FIXTURE)
-
-xtest-all: $(addprefix xtest-, $(TEST_SUITES))
+# e.g. `make xtest-System.Collections.Tests COREFX_ROOT=/prj/corefx`
+# '-parallel none -verbose' for debug, `-parallel all` to run all tests in parallel
+# we need COREFX_ROOT only for xunit.console.deps.json and CoreFx.Private.TestUtilities.dll
+xtest-%: prepare check-env dl-test-assets
+       ln -sf $(CURDIR)/sdk/$(NETCORESDK_VERSION)/System.Text.Encoding.CodePages.dll assets/extracted/$*/
+       cp $(COREFX_BINDIR)/runtime/netcoreapp-OSX-Debug-x64/xunit.console.deps.json assets/extracted/$*/
+       cp $(COREFX_BINDIR)/runtime/netcoreapp-OSX-Debug-x64/CoreFx.Private.TestUtilities.dll assets/extracted/$*/
+       sed -i -e 's/9.9.9/$(VERSION)/g' assets/extracted/$*/*.runtimeconfig.json
+       cd assets/extracted/$* && \
+       COMPlus_DebugWriteToStdErr=1 $(CURDIR)/./dotnet --fx-version "$(VERSION)" xunit.console.dll $*.dll \
+               -notrait category=outerloop -notrait category=nonosxtests -notrait category=failing \
+               -notrait category=nonnetcoreapptests -noappdomain -noshadow -parallel all -verbose \
+               -html ../../../TestResult-$*.html -nunit ../../TestResult-$*-netcore-xunit.xml \
+               $(shell if [ -a $(CURDIR)/excludes-$*.rsp ]; then grep -v '^#\|^$$' $(CURDIR)/excludes-$*.rsp; fi;) \
+               $(FIXTURE) || true
+
+# these tests won't be included in `xtestall` (some of them crash runtime, some hang)
+EXCLUDED_COREFX_TESTS = \
+       Microsoft.CSharp.Tests \
+       Microsoft.XmlSerializer.Generator.Tests \
+       System.ComponentModel.Composition.Tests \
+       System.Diagnostics.StackTrace.Tests \
+       System.Diagnostics.Tracing.Tests \
+       System.Dynamic.Runtime.Tests \
+       System.IO.Ports.Tests \
+       System.Linq.Expressions.Tests \
+       System.Linq.Queryable.Tests \
+       System.Net.Http.Functional.Tests \
+       System.Net.HttpListener.Tests \
+       System.Net.NameResolution.Functional.Tests \
+       System.Net.Sockets.Tests \
+       System.Reflection.Tests \
+       System.Runtime.InteropServices.Tests \
+       System.Runtime.Serialization.Formatters.Tests \
+       System.Runtime.Serialization.Json.Tests \
+       System.Runtime.Serialization.Xml.Tests \
+       System.Threading.Timer.Tests \
+       System.Xml.XmlSerializer.Tests \
+       System.Xml.Xsl.XslCompiledTransformApi.Tests
+
+xtestall: $(foreach workingtest, $(foreach test, $(wildcard assets/extracted/*), \
+               $(filter-out $(EXCLUDED_COREFX_TESTS), $(notdir $(test)))), $(addprefix xtest-, $(workingtest)))
 
 FEED_BASE_URL = https://dotnetfeed.blob.core.windows.net/dotnet-core
 TEST_ASSETS_URL = https://dotnetfeed.blob.core.windows.net/dotnet-core/corefx-tests/4.6.0-preview4.19156.10/OSX.x64/netcoreapp/corefx-test-assets.xml

diff --git a/src/mono/netcore/dl-test-assets.py b/src/mono/netcore/dl-test-assets.py
index a1e28c6..357da1c 100644 (file)
--- a/src/mono/netcore/dl-test-assets.py
+++ b/src/mono/netcore/dl-test-assets.py
@@ -1,8 +1,10 @@
 #!/usr/bin/env python
 
 import sys
+import os
 import subprocess
 import xml.etree.ElementTree as ET
+import zipfile
 
 if len(sys.argv) < 4:
     print("Usage: dl-test-assets.py <path to assets.xml> <base url> <output dir>")
@@ -17,9 +19,15 @@ root = tree.getroot()
 for elem in root:
     if elem.tag != "Blob":
         continue
-    print elem.attrib ["Id"]
-    res = subprocess.call (["wget", "-N", "-P", outdir, base_url + "/" + elem.attrib ["Id"]])
+    id = elem.attrib ["Id"]
+    filename = os.path.basename(id)
+    # System.IO.Compression.* have a lot of test files (>100mb)
+    if "System.IO.Compression" in filename:
+        continue
+    print filename
+    res = subprocess.call (["wget", "-N", "-P", outdir, base_url + "/" + id])
     if res != 0:
-        print ("Download failed.")
+        print ("Download failed for " + id)
         sys.exit (1)
-
+    with zipfile.ZipFile(outdir + "/" + filename) as zf:
+        zf.extractall(outdir + "/extracted/" + filename[:-4])

diff --git a/src/mono/netcore/excludes-System.Diagnostics.Process.Tests.rsp b/src/mono/netcore/excludes-System.Diagnostics.Process.Tests.rsp
new file mode 100644 (file)
index 0000000..47f608e
--- /dev/null
+++ b/src/mono/netcore/excludes-System.Diagnostics.Process.Tests.rsp
@@ -0,0 +1 @@
+-nomethod System.Diagnostics.Tests.ProcessTests.*
\ No newline at end of file

diff --git a/src/mono/netcore/excludes-System.Reflection.Emit.ILGeneration.Tests.rsp b/src/mono/netcore/excludes-System.Reflection.Emit.ILGeneration.Tests.rsp
new file mode 100644 (file)
index 0000000..7849087
--- /dev/null
+++ b/src/mono/netcore/excludes-System.Reflection.Emit.ILGeneration.Tests.rsp
@@ -0,0 +1 @@
+-noclass System.Reflection.Emit.Tests.CustomAttributeBuilderTests
\ No newline at end of file

diff --git a/src/mono/netcore/excludes-System.Runtime.Tests.rsp b/src/mono/netcore/excludes-System.Runtime.Tests.rsp
index 19a43d5..8728b2c 100644 (file)
--- a/src/mono/netcore/excludes-System.Runtime.Tests.rsp
+++ b/src/mono/netcore/excludes-System.Runtime.Tests.rsp
@@ -101,3 +101,4 @@
 -noclass System.Tests.ArgIteratorTests
 -nomethod System.Tests.ActivatorTests.CreateInstance_InvalidType_ThrowsNotSupportedException
 -nomethod System.Tests.ArrayTests.CreateInstance_NotSupportedType_ThrowsNotSupportedException
+-noclass System.Reflection.Tests.PointerTests

diff --git a/src/mono/netcore/excludes-System.Threading.Tests.rsp b/src/mono/netcore/excludes-System.Threading.Tests.rsp
new file mode 100644 (file)
index 0000000..45b0e99
--- /dev/null
+++ b/src/mono/netcore/excludes-System.Threading.Tests.rsp
@@ -0,0 +1,2 @@
+System.Threading.Tests.MutexTests.CrossProcess_NamedMutex_ProtectedFileAccessAtomic
+System.Threading.Tests.SemaphoreSlimTests.RunSemaphoreSlimTest1_WaitAsync_NegativeCases

diff --git a/src/mono/netcore/excludes-System.Xml.XmlSchemaSet.Tests.rsp b/src/mono/netcore/excludes-System.Xml.XmlSchemaSet.Tests.rsp
new file mode 100644 (file)
index 0000000..0790623
--- /dev/null
+++ b/src/mono/netcore/excludes-System.Xml.XmlSchemaSet.Tests.rsp
@@ -0,0 +1 @@
+-nomethod System.Xml.Tests.TC_SchemaSet_Misc.*