From: Stephen Toub Date: Tue, 19 Mar 2019 19:12:17 +0000 (-0400) Subject: Change Process.MaxWorkingSet to prefer cgroup data (dotnet/corefx#35645) X-Git-Tag: submit/tizen/20210909.063632~11031^2~2145 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f5899de7dbb92e7d20c54e9812f288d908091706;p=platform%2Fupstream%2Fdotnet%2Fruntime.git Change Process.MaxWorkingSet to prefer cgroup data (dotnet/corefx#35645) Commit migrated from https://github.com/dotnet/corefx/commit/dbcb7528847402325dabe40b9f447534d8a0617d --- diff --git a/src/libraries/Common/src/Interop/Linux/cgroups/Interop.cgroups.cs b/src/libraries/Common/src/Interop/Linux/cgroups/Interop.cgroups.cs new file mode 100644 index 0000000..0ffd4d7 --- /dev/null +++ b/src/libraries/Common/src/Interop/Linux/cgroups/Interop.cgroups.cs @@ -0,0 +1,225 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Buffers.Text; +using System.Diagnostics; +using System.IO; + +internal static partial class Interop +{ + internal static partial class cgroups + { + /// Path to mountinfo file in procfs for the current process. + private const string ProcMountInfoFilePath = "/proc/self/mountinfo"; + /// Path to cgroup directory in procfs for the current process. + private const string ProcCGroupFilePath = "/proc/self/cgroup"; + + /// Path to the found cgroup location, or null if it couldn't be found. + internal static readonly string s_cgroupMemoryPath = FindCGroupPath("memory"); + /// Path to the found cgroup memory limit_in_bytes path, or null if it couldn't be found. + private static readonly string s_cgroupMemoryLimitPath = s_cgroupMemoryPath != null ? s_cgroupMemoryPath + "/memory.limit_in_bytes" : null; + + /// Tries to read the memory limit from the cgroup memory location. + /// The read limit, or 0 if it couldn't be read. + /// true if the limit was read successfully; otherwise, false. + public static bool TryGetMemoryLimit(out ulong limit) + { + string path = s_cgroupMemoryLimitPath; + + if (path != null && + TryReadMemoryValueFromFile(path, out limit)) + { + return true; + } + + limit = 0; + return false; + } + + /// Tries to parse a memory limit from the specified file. + /// The path to the file to parse. + /// The parsed result, or 0 if it couldn't be parsed. + /// true if the value was read successfully; otherwise, false. + private static bool TryReadMemoryValueFromFile(string path, out ulong result) + { + if (File.Exists(path)) + { + try + { + byte[] bytes = File.ReadAllBytes(path); + if (Utf8Parser.TryParse(bytes, out ulong ulongValue, out int bytesConsumed)) + { + // If we successfully parsed the number, see if there's a K, M, or G + // multiplier value immediately following. + ulong multiplier = 1; + if (bytesConsumed < bytes.Length) + { + switch (bytes[bytesConsumed]) + { + + case (byte)'k': + case (byte)'K': + multiplier = 1024; + break; + + case (byte)'m': + case (byte)'M': + multiplier = 1024 * 1024; + break; + + case (byte)'g': + case (byte)'G': + multiplier = 1024 * 1024 * 1024; + break; + } + } + + result = checked(ulongValue * multiplier); + return true; + } + } + catch (Exception e) + { + Debug.Fail($"Failed to read \"{path}\": {e}"); + } + } + + result = 0; + return false; + } + + /// Find the cgroup path for the specified subsystem. + /// The subsystem, e.g. "memory". + /// The cgroup path if found; otherwise, null. + private static string FindCGroupPath(string subsystem) + { + if (TryFindHierarchyMount(subsystem, out string hierarchyRoot, out string hierarchyMount) && + TryFindCGroupPathForSubsystem(subsystem, out string cgroupPathRelativeToMount)) + { + // For a host cgroup, we need to append the relative path. + // In a docker container, the root and relative path are the same and we don't need to append. + return (hierarchyRoot != cgroupPathRelativeToMount) ? + hierarchyMount + cgroupPathRelativeToMount : + hierarchyMount; + } + + return null; + } + + /// Find the cgroup mount information for the specified subsystem. + /// The subsystem, e.g. "memory". + /// The path of the directory in the filesystem which forms the root of this mount; null if not found. + /// The path of the mount point relative to the process's root directory; null if not found. + /// true if the mount was found; otherwise, null. + private static bool TryFindHierarchyMount(string subsystem, out string root, out string path) + { + if (File.Exists(ProcMountInfoFilePath)) + { + try + { + using (var reader = new StreamReader(ProcMountInfoFilePath)) + { + string line; + while ((line = reader.ReadLine()) != null) + { + // Look for an entry that has cgroup as the "filesystem type" + // and that has options containing the specified subsystem. + // See man page for /proc/[pid]/mountinfo for details, e.g.: + // (1)(2)(3) (4) (5) (6) (7) (8) (9) (10) (11) + // 36 35 98:0 /mnt1 /mnt2 rw,noatime master:1 - ext3 /dev/root rw,errors=continue + // but (7) is optional and could exist as multiple fields; the (8) separator marks + // the end of the optional values. + + const string Separator = " - "; + int endOfOptionalFields = line.IndexOf(Separator); + if (endOfOptionalFields == -1) + { + // Malformed line. + continue; + } + + string postSeparatorLine = line.Substring(endOfOptionalFields + Separator.Length); + string[] postSeparatorlineParts = postSeparatorLine.Split(' '); + if (postSeparatorlineParts.Length < 3) + { + // Malformed line. + continue; + } + + if (postSeparatorlineParts[0] != "cgroup" || + Array.IndexOf(postSeparatorlineParts[2].Split(','), subsystem) < 0) + { + // Not the relevant entry. + continue; + } + + // Found the relevant entry. Extract the mount root and path. + string[] lineParts = line.Substring(0, endOfOptionalFields).Split(' '); + root = lineParts[3]; + path = lineParts[4]; + return true; + } + } + } + catch (Exception e) + { + Debug.Fail($"Failed to read or parse \"{ProcMountInfoFilePath}\": {e}"); + } + } + + root = null; + path = null; + return false; + } + + /// Find the cgroup relative path for the specified subsystem. + /// The subsystem, e.g. "memory". + /// The found path, or null if it couldn't be found. + /// + private static bool TryFindCGroupPathForSubsystem(string subsystem, out string path) + { + if (File.Exists(ProcCGroupFilePath)) + { + try + { + using (var reader = new StreamReader(ProcCGroupFilePath)) + { + string line; + while ((line = reader.ReadLine()) != null) + { + // Find the first entry that has the subsystem listed in its controller + // list. See man page for cgroups for /proc/[pid]/cgroups format, e.g: + // hierarchy-ID:controller-list:cgroup-path + // 5:cpuacct,cpu,cpuset:/daemons + + string[] lineParts = line.Split(':'); + if (lineParts.Length != 3) + { + // Malformed line. + continue; + } + + if (Array.IndexOf(lineParts[1].Split(','), subsystem) < 0) + { + // Not the relevant entry. + continue; + } + + path = lineParts[2]; + return true; + } + } + } + catch (Exception e) + { + Debug.Fail($"Failed to read or parse \"{ProcMountInfoFilePath}\": {e}"); + } + } + + path = null; + return false; + } + } +} diff --git a/src/libraries/System.Diagnostics.Process/src/System.Diagnostics.Process.csproj b/src/libraries/System.Diagnostics.Process/src/System.Diagnostics.Process.csproj index 7f02d34..73a2179 100644 --- a/src/libraries/System.Diagnostics.Process/src/System.Diagnostics.Process.csproj +++ b/src/libraries/System.Diagnostics.Process/src/System.Diagnostics.Process.csproj @@ -367,6 +367,9 @@ + + Common\Interop\Linux\Interop.cgroups.cs + Common\Interop\Linux\Interop.ProcFsStat.cs diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Linux.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Linux.cs index f0f7d76..6c04a33 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Linux.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Linux.cs @@ -202,7 +202,14 @@ namespace System.Diagnostics private void GetWorkingSetLimits(out IntPtr minWorkingSet, out IntPtr maxWorkingSet) { minWorkingSet = IntPtr.Zero; // no defined limit available - ulong rsslim = GetStat().rsslim; + + // For max working set, try to respect container limits by reading + // from cgroup, but if it's unavailable, fall back to reading from procfs. + EnsureState(State.HaveNonExitedId); + if (!Interop.cgroups.TryGetMemoryLimit(out ulong rsslim)) + { + rsslim = GetStat().rsslim; + } // rsslim is a ulong, but maxWorkingSet is an IntPtr, so we need to cap rsslim // at the max size of IntPtr. This often happens when there is no configured diff --git a/src/libraries/System.Runtime.InteropServices.RuntimeInformation/tests/DescriptionNameTests.cs b/src/libraries/System.Runtime.InteropServices.RuntimeInformation/tests/DescriptionNameTests.cs index 821c112..8c2acaf 100644 --- a/src/libraries/System.Runtime.InteropServices.RuntimeInformation/tests/DescriptionNameTests.cs +++ b/src/libraries/System.Runtime.InteropServices.RuntimeInformation/tests/DescriptionNameTests.cs @@ -2,8 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Collections; +using System.Diagnostics; using System.IO; -using System.Reflection; +using System.Text; using Xunit; namespace System.Runtime.InteropServices.RuntimeInformationTests @@ -13,24 +15,24 @@ namespace System.Runtime.InteropServices.RuntimeInformationTests [Fact] public void DumpRuntimeInformationToConsole() { - // Not really a test, but useful to dump to the log to - // sanity check that the test run or CI job - // was actually run on the OS that it claims to be on + // Not really a test, but useful to dump a variety of information to the test log to help + // debug environmental issues, in particular in CI + string dvs = PlatformDetection.GetDistroVersionString(); string osd = RuntimeInformation.OSDescription.Trim(); string osv = Environment.OSVersion.ToString(); string osa = RuntimeInformation.OSArchitecture.ToString(); - string pra = RuntimeInformation.ProcessArchitecture.ToString(); - string frd = RuntimeInformation.FrameworkDescription.Trim(); + Console.WriteLine($"### OS: Distro={dvs} Description={osd} Version={osv} Arch={osa}"); + string lcr = PlatformDetection.LibcRelease; string lcv = PlatformDetection.LibcVersion; - - Console.WriteLine($@"### CONFIGURATION: {dvs} OS={osd} OSVer={osv} OSArch={osa} Arch={pra} Framework={frd} LibcRelease={lcr} LibcVersion={lcv}"); + Console.WriteLine($"### LIBC: Release={lcr} Version={lcv}"); + + Console.WriteLine($"### FRAMEWORK: Version={Environment.Version} Description={RuntimeInformation.FrameworkDescription.Trim()}"); if (!PlatformDetection.IsNetNative) { string binariesLocation = Path.GetDirectoryName(typeof(object).Assembly.Location); - Console.WriteLine("location: " + binariesLocation); string binariesLocationFormat = PlatformDetection.IsInAppContainer ? "Unknown" : new DriveInfo(binariesLocation).DriveFormat; Console.WriteLine($"### BINARIES: {binariesLocation} (drive format {binariesLocationFormat})"); } @@ -40,6 +42,100 @@ namespace System.Runtime.InteropServices.RuntimeInformationTests Console.WriteLine($"### TEMP PATH: {tempPathLocation} (drive format {tempPathLocationFormat})"); Console.WriteLine($"### CURRENT DIRECTORY: {Environment.CurrentDirectory}"); + + string cgroupsLocation = Interop.cgroups.s_cgroupMemoryPath; + if (cgroupsLocation != null) + { + Console.WriteLine($"### CGROUPS MEMORY: {cgroupsLocation}"); + } + + Console.WriteLine($"### ENVIRONMENT VARIABLES"); + foreach (DictionaryEntry envvar in Environment.GetEnvironmentVariables()) + { + Console.WriteLine($"###\t{envvar.Key}: {envvar.Value}"); + } + + using (Process p = Process.GetCurrentProcess()) + { + var sb = new StringBuilder(); + sb.AppendLine("### PROCESS INFORMATION:"); + sb.AppendFormat($"###\tArchitecture: {RuntimeInformation.ProcessArchitecture.ToString()}").AppendLine(); + foreach (string prop in new string[] + { + #pragma warning disable 0618 // some of these Int32-returning properties are marked obsolete + nameof(p.BasePriority), + nameof(p.HandleCount), + nameof(p.Id), + nameof(p.MachineName), + nameof(p.MainModule), + nameof(p.MainWindowHandle), + nameof(p.MainWindowTitle), + nameof(p.MaxWorkingSet), + nameof(p.MinWorkingSet), + nameof(p.NonpagedSystemMemorySize), + nameof(p.NonpagedSystemMemorySize64), + nameof(p.PagedMemorySize), + nameof(p.PagedMemorySize64), + nameof(p.PagedSystemMemorySize), + nameof(p.PagedSystemMemorySize64), + nameof(p.PeakPagedMemorySize), + nameof(p.PeakPagedMemorySize64), + nameof(p.PeakVirtualMemorySize), + nameof(p.PeakVirtualMemorySize64), + nameof(p.PeakWorkingSet), + nameof(p.PeakWorkingSet64), + nameof(p.PriorityBoostEnabled), + nameof(p.PriorityClass), + nameof(p.PrivateMemorySize), + nameof(p.PrivateMemorySize64), + nameof(p.PrivilegedProcessorTime), + nameof(p.ProcessName), + nameof(p.ProcessorAffinity), + nameof(p.Responding), + nameof(p.SessionId), + nameof(p.StartTime), + nameof(p.TotalProcessorTime), + nameof(p.UserProcessorTime), + nameof(p.VirtualMemorySize), + nameof(p.VirtualMemorySize64), + nameof(p.WorkingSet), + nameof(p.WorkingSet64), + #pragma warning restore 0618 + }) + { + sb.Append($"###\t{prop}: "); + try + { + sb.Append(p.GetType().GetProperty(prop).GetValue(p)); + } + catch (Exception e) + { + sb.Append($"(Exception: {e.Message})"); + } + sb.AppendLine(); + } + Console.WriteLine(sb.ToString()); + } + + if (osd.Contains("Linux")) + { + // Dump several procfs files + foreach (string path in new string[] { "/proc/self/mountinfo", "/proc/self/cgroup", "/proc/self/limits" }) + { + Console.WriteLine($"### CONTENTS OF \"{path}\":"); + try + { + using (Process cat = Process.Start("cat", path)) + { + cat.WaitForExit(); + } + } + catch (Exception e) + { + Console.WriteLine($"###\t(Exception: {e.Message})"); + } + } + } } [Fact] diff --git a/src/libraries/System.Runtime.InteropServices.RuntimeInformation/tests/System.Runtime.InteropServices.RuntimeInformation.Tests.csproj b/src/libraries/System.Runtime.InteropServices.RuntimeInformation/tests/System.Runtime.InteropServices.RuntimeInformation.Tests.csproj index d308e96..7594980 100644 --- a/src/libraries/System.Runtime.InteropServices.RuntimeInformation/tests/System.Runtime.InteropServices.RuntimeInformation.Tests.csproj +++ b/src/libraries/System.Runtime.InteropServices.RuntimeInformation/tests/System.Runtime.InteropServices.RuntimeInformation.Tests.csproj @@ -8,5 +8,8 @@ + + Common\Interop\Linux\Interop.cgroups.cs +