The bulk of this PR is a cherry-pick of commit fa02660 that shipped in
release/1.1.0, updating the build system support for PGO to support
consuming PGDs properly during release builds on Windows.
Also included are the following new changes:
* Skip restore of opdata if the requisite project.json is missing
* If the optdata package restore fails, fail the build.
* Add new build option: 'skiprestoreoptdata'
Note: This change doesn't by itself enable PGO in master yet, because
training data (optdata packages) for master don't exist on myget yet.
However, with these changes, the only step remaining to enable PGO
optimizations is to add a project.json referencing the correct optdata
package.
Commit migrated from https://github.com/dotnet/coreclr/commit/
1a1fdb8fb0839832b64248582082c90e166b74b5
set "__PackagesDir=%__ProjectDir%\packages"
set "__RootBinDir=%__ProjectDir%\bin"
set "__LogsDir=%__RootBinDir%\Logs"
+set "__OptDataVersion="
set __BuildAll=
set __BuildTests=1
set __BuildPackages=1
set __BuildNativeCoreLib=1
+set __RestoreOptData=1
:Arg_Loop
if "%1" == "" goto ArgsDone
if /i "%1" == "skipnative" (set __BuildNative=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
if /i "%1" == "skiptests" (set __BuildTests=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
if /i "%1" == "skipbuildpackages" (set __BuildPackages=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "skiprestoreoptdata" (set __RestoreOptData=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
if /i "%1" == "usenmakemakefiles" (set __NMakeMakefiles=1&set __ConfigureOnly=1&set __BuildNative=1&set __BuildNativeCoreLib=0&set __BuildCoreLib=0&set __BuildTests=0&set __BuildPackages=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
if /i "%1" == "buildjit32" (set __BuildJit32="-DBUILD_JIT32=1"&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
if /i "%1" == "pgoinstrument" (set __PgoInstrument=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
REM =========================================================================================
REM ===
+REM === Restore optimization profile data
+REM ===
+REM =========================================================================================
+
+REM Parse the package version out of project.json so that we can pass it on to CMake
+where /q python || (
+ echo %__MsgPrefix%Error: Python not found on PATH, please make sure that it is installed.
+ exit /b 1
+)
+set OptDataProjectJsonPath=%__ProjectDir%\src\.nuget\optdata\project.json
+if EXIST "%OptDataProjectJsonPath%" (
+ for /f "tokens=*" %%s in ('python "%__ProjectDir%\extract-from-json.py" -rf "%OptDataProjectJsonPath%" dependencies optimization.PGO.CoreCLR') do @(
+ set __OptDataVersion=%%s
+ )
+)
+
+if %__RestoreOptData% EQU 1 (
+ echo %__MsgPrefix%Restoring the OptimizationData Package
+ @call %__ProjectDir%\run.cmd sync -optdata
+ if not !errorlevel! == 0 (
+ echo %__MsgPrefix%Error: Failed to restore the optimization data package.
+ exit /b 1
+ )
+)
+
+REM =========================================================================================
+REM ===
REM === Build the CLR VM
REM ===
REM =========================================================================================
echo %__MsgPrefix%Regenerating the Visual Studio solution
pushd "%__IntermediatesDir%"
- set __ExtraCmakeArgs=!___SDKVersion! "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%"
+ set __ExtraCmakeArgs=!___SDKVersion! "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%" "-DCLR_CMAKE_OPTDATA_VERSION=%__OptDataVersion%"
call "%__SourceDir%\pal\tools\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__BuildArch% %__BuildJit32% %__BuildStandaloneGC% !__ExtraCmakeArgs!
@if defined _echo @echo on
popd
pushd "%__CrossCompIntermediatesDir%"
set __CMakeBinDir=%__CrossComponentBinDir%
set "__CMakeBinDir=!__CMakeBinDir:\=/!"
- set __ExtraCmakeArgs="-DCLR_CROSS_COMPONENTS_BUILD=1" "-DCLR_CMAKE_TARGET_ARCH=%__BuildArch%" "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%"
+ set __ExtraCmakeArgs="-DCLR_CROSS_COMPONENTS_BUILD=1" "-DCLR_CMAKE_TARGET_ARCH=%__BuildArch%" "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%" "-DCLR_CMAKE_OPTDATA_VERSION=%__OptDataVersion%"
call "%__SourceDir%\pal\tools\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__CrossArch% !__ExtraCmakeArgs!
@if defined _echo @echo on
popd
echo skipnative: skip building native components ^(default: native components are built^).
echo skiptests: skip building tests ^(default: tests are built^).
echo skipbuildpackages: skip building nuget packages ^(default: packages are built^).
+echo skiprestoreoptdata: skip restoring optimization data used by profile-based optimizations.
echo buildstandalonegc: builds the GC in a standalone mode.
echo -skiprestore: skip restoring packages ^(default: packages are restored during build^).
echo -disableoss: Disable Open Source Signing for System.Private.CoreLib.
<Delete Files="$(BinDir)System.Private.CoreLib.*" />
</Target>
+ <PropertyGroup>
+ <OptDataProjectJson>$(SourceDir).nuget/optdata/project.json</OptDataProjectJson>
+ <OptDataPackageFeed>https://dotnet.myget.org/F/dotnet-core-optimization-data/api/v3/index.json</OptDataPackageFeed>
+ </PropertyGroup>
+ <Target Name="RestoreOptData">
+ <Exec Condition="Exists('$(OptDataProjectJson)')" Command="$(DnuRestoreCommand) "$(OptDataProjectJson)" --source "$(OptDataPackageFeed)"" />
+ </Target>
+
<Target Name="RestoreNETCorePlatforms" AfterTargets="Build" Condition="'$(RestoreDuringBuild)'=='true'">
<Exec Command="$(DnuRestoreCommand) "$(SourceDir).nuget/init/project.json" --source https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" />
</Target>
echo "skipmscorlib - do not build mscorlib.dll."
echo "skiptests - skip the tests in the 'tests' subdirectory."
echo "skipnuget - skip building nuget packages."
+ echo "skiprestoreoptdata - skip restoring optimization data used by profile-based optimizations."
echo "portableLinux - build for Portable Linux Distribution"
echo "verbose - optional argument to enable verbose build output."
echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
}
+restore_optdata()
+{
+ # if msbuild is not supported, then set __SkipRestoreOptData to 1
+ if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipRestoreOptData=1; fi
+ if [ $__SkipRestoreOptData == 0 ]; then
+ echo "Restoring the OptimizationData package"
+ "$__ProjectRoot/run.sh" sync -optdata
+ if [ $? != 0 ]; then
+ echo "Failed to restore the optimization data package."
+ exit 1
+ fi
+ fi
+}
generate_event_logging_sources()
{
fi
fi
+
pushd "$intermediatesForBuild"
# Regenerate the CMake solution
echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $platformArch $__BuildType $__CodeCoverage $__IncludeTests $generator $extraCmakeArguments $__cmakeargs"
fi
fi
- __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument"
+ __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__OptDataVersion"
build_native $__SkipCrossArchBuild "$__CrossArch" "$__CrossCompIntermediatesDir" "$__ExtraCmakeArgs" "cross-architecture component"
# restore ROOTFS_DIR, CROSSCOMPONENT, and CROSSCOMPILE
__SkipNuget=0
__SkipCoreCLR=0
__SkipMSCorLib=0
+__SkipRestoreOptData=0
__CrossBuild=0
__ClangMajorVersion=0
__ClangMinorVersion=0
__DoCrossArchBuild=0
__PortableLinux=0
__msbuildonunsupportedplatform=0
+__OptDataVersion=""
while :; do
if [ $# -le 0 ]; then
__SkipGenerateVersion=1
;;
+ skiprestoreoptdata)
+ __SkipRestoreOptData=1
+ ;;
+
includetests)
;;
fi
fi
+# Parse the optdata package version from its project.json file
+optDataProjectJsonPath="$__ProjectRoot/src/.nuget/optdata/project.json"
+if [ -f $optDataProjectJsonPath ]; then
+ __OptDataVersion=$("$__ProjectRoot/extract-from-json.py" -rf $optDataProjectJsonPath dependencies optimization.PGO.CoreCLR)
+fi
+
# init the target distro name
initTargetDistroRid
# Check prereqs.
check_prereqs
+# Restore the package containing profile counts for profile-guided optimizations
+restore_optdata
+
# Generate event logging infrastructure sources
generate_event_logging_sources
# Build the coreclr (native) components.
-__ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument"
+__ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__OptDataVersion"
build_native $__SkipCoreCLR "$__BuildArch" "$__IntermediatesDir" "$__ExtraCmakeArgs" "CoreCLR component"
# Build cross-architecture components
"values": [],
"defaultValue": ""
},
+ "RestoreOptData": {
+ "description": "MsBuild target that restores optimization profile data.",
+ "valueType": "target",
+ "values": [],
+ "defaultValue": ""
+ },
"RestoreDuringBuild": {
"description": "Enables/disables package restore.",
"valueType": "property",
"RestoreNETCorePlatforms": "default"
}
},
+ "optdata": {
+ "description": "Restores optimization profile data for the repository.",
+ "settings": {
+ "Project": "./build.proj",
+ "RestoreDuringBuild": true,
+ "RestoreOptData": "default"
+ }
+ },
"ab": {
"description": "Downloads the latests product packages from Azure. The values for '-AzureAccount' and '-AzureToken' are required",
"settings": {
--- /dev/null
+#!/usr/bin/python
+
+import argparse
+import json
+import sys
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="""Extracts information from a json file by navigating the JSON object using a
+ sequence of property accessors and returning the JSON subtree, or the raw data, found
+ at that location."""
+ )
+
+ parser.add_argument(
+ '-f', '--file',
+ metavar='<project.json>',
+ help="Path to project.json file to parse",
+ required=True,
+ )
+
+ parser.add_argument(
+ 'property',
+ metavar='property_name',
+ help="""Name of property to extract using object notation.
+ Pass multiple values to drill down into nested objects (in order).""",
+ nargs='*',
+ )
+
+ parser.add_argument(
+ '-r', '--raw',
+ help="""Dumps the raw object found at the requested location.
+ If omitted, returns a JSON formatted object instead.""",
+ action='store_true',
+ default=False
+ )
+
+ return parser.parse_args()
+
+def main():
+ args = parse_args()
+
+ with open(args.file) as json_file:
+ selected_property = json.load(json_file)
+
+ for prop in args.property:
+ selected_property = selected_property[prop]
+
+ if args.raw:
+ print(selected_property)
+ else:
+ print(json.dumps(selected_property))
+
+ return 0
+
+if __name__ == "__main__":
+ sys.exit(main())
set(ProfileFileName "${TargetName}.pgd")
endif(WIN32)
+ set(CLR_CMAKE_OPTDATA_PACKAGEWITHRID "optimization.${CLR_CMAKE_TARGET_OS}-${CLR_CMAKE_TARGET_ARCH}.PGO.CoreCLR")
file(TO_NATIVE_PATH
- "${CLR_CMAKE_PACKAGES_DIR}/Microsoft.DotNet.OptimizationData.Coreclr/${CLR_CMAKE_TARGET_OS}.${CLR_CMAKE_TARGET_ARCH}/${ProfileFileName}"
+ "${CLR_CMAKE_PACKAGES_DIR}/${CLR_CMAKE_OPTDATA_PACKAGEWITHRID}/${CLR_CMAKE_OPTDATA_VERSION}/data/${ProfileFileName}"
ProfilePath
)