Documentation/project-docs/clr-configuration-knobs.csx

   1 // This script generates documentation about the various configuration options that
   2 // are available and how to use them.
   3
   4 using System;
   5 using System.Collections.Generic;
   6 using System.Diagnostics;
   7 using System.Globalization;
   8 using System.IO;
   9 using System.Linq;
  10 using System.Text;
  11 using System.Text.RegularExpressions;
  12 using static System.Console;
  13
  14 char dirSep = Path.DirectorySeparatorChar;
  15 var clrConfig = $"..{dirSep}..{dirSep}src{dirSep}inc{dirSep}clrconfigvalues.h";
  16 var jitConfig = $"..{dirSep}..{dirSep}src{dirSep}jit{dirSep}jitconfigvalues.h";
  17 Console.Out.NewLine = "\n";
  18 Console.Error.NewLine = "\n";
  19
  20 public struct Knob
  21 {
  22     private string _name;
  23     private string _class;
  24
  25     public bool Retail;
  26     public string Category;
  27     public string Type;
  28     public string DefaultValue;
  29     public string Description;
  30     public string Flags;
  31     public int Line;
  32     public string File;
  33
  34     public string Name
  35     {
  36         get => _name;
  37         set
  38         {
  39             value = value ?? String.Empty;
  40             int indexOfUnderscore = value.IndexOf("_");
  41             if (indexOfUnderscore > 0 && (value.Contains("INTERNAL") || value.Contains("EXTERNAL") || value.Contains("UNSUPPORTED")))
  42             {
  43                 _name = value.Substring(indexOfUnderscore + 1);
  44             }
  45             else
  46             {
  47                 _name = value;
  48             }
  49         }
  50     }
  51
  52     public string Class
  53     {
  54         get => _class;
  55         set
  56         {
  57             value = value ?? String.Empty;
  58             int indexOfUnderscore = value.IndexOf("_");
  59             if (indexOfUnderscore > 0 && (value.Contains("INTERNAL") || value.Contains("EXTERNAL") || value.Contains("UNSUPPORTED")))
  60             {
  61                 _class = value.Substring(0, indexOfUnderscore);
  62             }
  63             else
  64             {
  65                 _class = value;
  66             }
  67         }
  68     }
  69
  70     public string Location => $"{File}:{Line}";
  71
  72     private const string StringType = "STRING";
  73     private const string DWORD_Type = "DWORD";
  74     private const string SpaceSeparatedValues = "SSV";
  75
  76     private static Regex s_cppString = new Regex("\"([^\"\\\\]|\\\\.)*\"", RegexOptions.Compiled);
  77
  78     enum Fields
  79     {
  80         Unknown = 0,
  81         Symbol,
  82         Name,
  83         DefaultValue,
  84         Description,
  85         LookupOptions
  86     }
  87
  88     public Knob(string line, bool isRetail, bool isClrConfigFile, StreamReader reader, ref int lineNum, out string nextLine)
  89     {
  90         nextLine = null;
  91
  92         _name = String.Empty;
  93         _class = String.Empty;
  94
  95         Retail = isRetail;
  96         Category = String.Empty;
  97         Type = String.Empty;
  98         DefaultValue = String.Empty;
  99         Description = String.Empty;
 100         Flags = String.Empty;
 101         Line = -1;
 102         File = isClrConfigFile ? "clrconfigvalues.h" : "jitconfigvalues.h";
 103
 104         string[] parts0 = null;
 105         string[] parts1 = null;
 106         string[] parts2 = null;
 107
 108         // Split on first open parenthesis
 109         parts0 = line.Split(new char[]{'('}, 2, StringSplitOptions.None);
 110
 111         if (parts0.Length > 1)
 112         {
 113             Fields[] numParts1 = null;
 114             switch(parts0[0])
 115             {
 116                 // CONFIG_DWORD_INFO(symbol, name, defaultValue, description)
 117                 case "CONFIG_DWORD_INFO":
 118                 case "RETAIL_CONFIG_DWORD_INFO":
 119                     numParts1 = new Fields[] { Fields.Symbol, Fields.Name, Fields.DefaultValue, Fields.Description };
 120                     parts1 = parts0[1].Split(new[] { ',' }, 4);
 121                     Type = DWORD_Type;
 122                     break;
 123
 124                 // CONFIG_DWORD_INFO_DIRECT_ACCESS(symbol, name, description)
 125                 case "CONFIG_DWORD_INFO_DIRECT_ACCESS":
 126                 case "RETAIL_CONFIG_DWORD_INFO_DIRECT_ACCESS":
 127                     numParts1 = new Fields[] { Fields.Symbol, Fields.Name, Fields.Description };
 128                     parts1 = parts0[1].Split(new[] { ',' }, 3);
 129                     Type = DWORD_Type;
 130                     break;
 131
 132                 // CONFIG_STRING_INFO(symbol, name, description)
 133                 case "CONFIG_STRING_INFO":
 134                 case "RETAIL_CONFIG_STRING_INFO":
 135                 case "CONFIG_STRING_INFO_DIRECT_ACCESS":
 136                 case "RETAIL_CONFIG_STRING_INFO_DIRECT_ACCESS":
 137                     numParts1 = new Fields[] { Fields.Symbol, Fields.Name, Fields.Description };
 138                     parts1 = parts0[1].Split(new[] { ',' }, 3);
 139                     Type = StringType;
 140                     break;
 141
 142                 // CONFIG_DWORD_INFO_EX(symbol, name, defaultValue, description, lookupOptions)
 143                 case "CONFIG_DWORD_INFO_EX":
 144                 case "RETAIL_CONFIG_DWORD_INFO_EX":
 145                     numParts1 = new Fields[] { Fields.Symbol, Fields.Name, Fields.DefaultValue, Fields.Description, Fields.LookupOptions };
 146                     parts1 = parts0[1].Split(new[] { ',' }, 4);
 147                     Type = DWORD_Type;
 148                     break;
 149
 150                 // CONFIG_STRING_INFO_EX(symbol, name, description, lookupOptions)
 151                 case "CONFIG_STRING_INFO_EX":
 152                 case "RETAIL_CONFIG_STRING_INFO_EX":
 153                     numParts1 = new Fields[] { Fields.Symbol, Fields.Name, Fields.Description, Fields.LookupOptions };
 154                     parts1 = parts0[1].Split(new[] { ',' }, 3);
 155                     Type = StringType;
 156                     break;
 157
 158                 // CONFIG_INTEGER(symbol, name, defaultValue) description in single line comments
 159                 case "CONFIG_INTEGER":
 160                     numParts1 = new Fields[] { Fields.Symbol, Fields.Name, Fields.DefaultValue };
 161                     parts1 = parts0[1].Split(new[] { ',' }, 3);
 162                     Type = DWORD_Type;
 163                     break;
 164
 165                 // CONFIG_STRING(symbol, name) description in single line comments
 166                 case "CONFIG_STRING":
 167                     numParts1 = new Fields[] { Fields.Symbol, Fields.Name };
 168                     parts1 = parts0[1].Split(new[] { ',' }, 2);
 169                     Type = StringType;
 170                     break;
 171
 172                 // CONFIG_METHODSET(symbol, name) description in single line comments
 173                 case "CONFIG_METHODSET":
 174                     numParts1 = new Fields[] { Fields.Symbol, Fields.Name };
 175                     parts1 = parts0[1].Split(new[] { ',' }, 2);
 176                     Type = SpaceSeparatedValues;
 177                     break;
 178
 179                 default:
 180                     throw new ArgumentException($"Unsupported C++ macro definition: {parts0[0]}", nameof(line));
 181             }
 182
 183             // Parse Description
 184             if (numParts1[parts1.Length - 1] == Fields.Description)
 185             {
 186                 var descMatch = s_cppString.Match(parts1[parts1.Length - 1]);
 187                 Description = descMatch.Value.Substring(1, descMatch.Length - 2);
 188
 189                 // Parse Flags if present
 190                 if (parts1.Length < numParts1.Length && numParts1[parts1.Length] == Fields.LookupOptions)
 191                 {
 192                     var tempFlags = parts1[parts1.Length - 1].Substring(descMatch.Index + descMatch.Length).TrimStart(' ', ',');
 193                     Flags = tempFlags.Replace("CLRConfig::", null).Replace(")", null).Replace("|", "\\|");
 194                 }
 195             }
 196
 197             // Parse Description in jitconfigvalues.h file which is in single line C++ comments
 198             else if (!isClrConfigFile)
 199             {
 200                 int commentIndex = line.IndexOf(" // ");
 201                 var description = commentIndex >= 0 ? line.Substring(commentIndex + 4) : String.Empty;
 202                 if (description.Length > 0)
 203                 {
 204                     nextLine = reader.ReadLine();
 205                     lineNum++;
 206                     while (nextLine != null)
 207                     {
 208                         var workLine = nextLine.Trim();
 209                         if (workLine.StartsWith("// "))
 210                         {
 211                             description += workLine.Substring(2).TrimEnd();
 212                             nextLine = reader.ReadLine();
 213                             lineNum++;
 214                         }
 215                         else
 216                         {
 217                             break;
 218                         }
 219                     }
 220                 }
 221                 Description = description;
 222             }
 223
 224             Description = Description.Replace("|", "\\|");
 225
 226             // Parse DefaultValue
 227             int indexOfDefaultValue = -1;
 228             if ((indexOfDefaultValue = Array.IndexOf<Fields>(numParts1, Fields.DefaultValue)) >= 0 && indexOfDefaultValue < parts1.Length)
 229             {
 230                 if (isClrConfigFile)
 231                 {
 232                     DefaultValue = parts1[indexOfDefaultValue].TrimEnd(')');
 233                 }
 234                 else
 235                 {
 236                     int indexOfCloseParenth = parts1[indexOfDefaultValue].IndexOf(")", StringComparison.Ordinal);
 237                     DefaultValue = parts1[indexOfDefaultValue].Substring(0, indexOfCloseParenth);
 238                 }
 239
 240                 DefaultValue = DefaultValue.Trim();
 241             }
 242
 243             // Parse Name
 244             var nameMatch = s_cppString.Match(parts1[1]);
 245             Name = nameMatch.Captures.Count > 0 ? nameMatch.Value.Substring(1, nameMatch.Length - 2) : String.Empty;
 246
 247             // Parse Class
 248             if (parts1[0].Length > 0)
 249             {
 250                 Class = Name.Length > 0 ? parts1[0].Replace(Name, null) : parts1[0];
 251                 Class = Class.TrimEnd('_');
 252             }
 253
 254             Retail = isRetail;
 255
 256             if (Flags == null)
 257             {
 258                 Flags = String.Empty;
 259             }
 260
 261         }
 262     }
 263
 264     public override string ToString()
 265     {
 266         return $"{{Knob: Name: {Name}, Category: {Category}, Retail: {Retail}, Class: {Class}, Type: {Type}, DefaultValue: {DefaultValue}, Description: {Description}, Flags: {Flags}, Location: {File}:{Line}}}";
 267     }
 268 }
 269
 270 public static void ParseConfigFile(string filePath, bool isClrConfigFile, SortedDictionary<string, SortedDictionary<string, Knob>> categorizedKnobsDictionary)
 271 {
 272     using (StreamReader clrReader = new StreamReader(filePath, new UTF8Encoding(false)))
 273     {
 274         SortedDictionary<string, Knob> knobsDictionary = null;
 275         string currentCategory = null;
 276
 277         int lineNumber = 1;
 278         string line = clrReader.ReadLine();
 279         while (line != null)
 280         {
 281             bool isRetail = false;
 282             string nextLine = null;
 283
 284             if (line.StartsWith("CONFIG_", StringComparison.Ordinal) || (isRetail = line.StartsWith("RETAIL_CONFIG_", StringComparison.Ordinal)))
 285             {
 286                 var clrKnob = new Knob(line, isRetail, isClrConfigFile, clrReader, ref lineNumber, out nextLine);
 287                 clrKnob.Category = currentCategory;
 288                 clrKnob.Line = lineNumber;
 289                 if (!knobsDictionary.ContainsKey(clrKnob.Name))
 290                 {
 291                     knobsDictionary.Add(clrKnob.Name, clrKnob);
 292                 }
 293                 else
 294                 {
 295                     Knob dupKnob = knobsDictionary[clrKnob.Name];
 296                     if (dupKnob.File != clrKnob.File && !dupKnob.Retail)
 297                     {
 298                         WriteLine($"Duplicate: {dupKnob.Location} {dupKnob.Name} -> {clrKnob.Location} {clrKnob.Name}");
 299                     }
 300                 }
 301             }
 302             else if (line.StartsWith("///", StringComparison.Ordinal) && (line = line.Trim()).Length > 3)
 303             {
 304                 line = line.Replace("///", null).Trim();
 305                 if (!categorizedKnobsDictionary.ContainsKey(line))
 306                 {
 307                     knobsDictionary = new SortedDictionary<string, Knob>();
 308                     categorizedKnobsDictionary.Add(line, knobsDictionary);
 309                     currentCategory = line;
 310                 }
 311                 else
 312                 {
 313                     knobsDictionary = categorizedKnobsDictionary[line];
 314                     currentCategory = line;
 315                 }
 316             }
 317
 318             if (nextLine != null)
 319             {
 320                 line = nextLine;
 321             }
 322             else
 323             {
 324                 line = clrReader.ReadLine();
 325                 lineNumber++;
 326             }
 327         }
 328     }
 329 }
 330
 331 public static class ConfigKnobsDoc
 332 {
 333     public static string IntroSection =
 334     "There are two primary ways to configure runtime behavior: CoreCLR hosts can pass in key-value string pairs during runtime initialization, or users can set special variables in the environment or registry. Today, the set of configuration options that can be set via the former method is relatively small, but moving forward, we expect to add more options there. Each set of options is described below.\n";
 335
 336     public static string HostConfigurationKnobsPart1 =
 337         "## Host Configuration Knobs\nThese can be passed in by a host during initialization. Note that the values are all passed in as strings, so if the type is boolean, the value would be the string \"true\" or \"false\", and if it's a numeric value, it would be in the form \"123\".\n";
 338
 339     public static string HostConfigurationKnobsPart2 =
 340         "\nName | Description | Type\n" +
 341         "-----|-------------|------\n" +
 342         "`System.GC.Concurrent` | Enable concurrent GC | boolean\n" +
 343         "`System.GC.Server` | Enable server GC | boolean\n" +
 344         "`System.GC.RetainVM` | Put segments that should be deleted on a standby list for future use instead of releasing them back to the OS | boolean\n" +
 345         "`System.Runtime.TieredCompilation` | Enable tiered compilation | boolean\n" +
 346         "`System.Threading.ThreadPool.MinThreads` | Override MinThreads for the ThreadPool worker pool | numeric\n" +
 347         "`System.Threading.ThreadPool.MaxThreads` | Override MaxThreads for the ThreadPool worker pool | numeric\n\n\n";
 348
 349     public static string ClrConfigSectionHeader =
 350         "## Environment/Registry Configuration Knobs\n";
 351
 352     public static string ClrConfigSectionInfo =
 353         "This table was machine-generated using `clr-configuration-knobs.csx` script from repository commit [GIT_SHORT_HASH](https://github.com/dotnet/coreclr/commit/GIT_LONG_HASH) on DATE_CREATED. It might be out of date. To generate latest documentation run `{dotnet} csi clr-configuration-knobs.csx` from this file directory.\n";
 354
 355     public static string ClrConfigSectionUsage =
 356         "When using these configurations from environment variables, the variables need to have the `COMPlus_` prefix in their names. e.g. To set DumpJittedMethods to 1, add the environment variable `COMPlus_DumpJittedMethods=1`.\n\nSee also [Setting configuration variables](../building/viewing-jit-dumps.md#setting-configuration-variables) for more information.\n";
 357
 358     public static string ClrConfigTableHeader =
 359         "\nName | Description | Type | Class | Default Value | Flags \n" +
 360         "-----|-------------|------|-------|---------------|-------\n";
 361
 362     public static string PalConfigurationKnobs =
 363         "## PAL Configuration Knobs\n" +
 364         "All the names below need to be prefixed by `COMPlus_`.\n\n" +
 365         "Name | Description | Type | Default Value\n" +
 366         "-----|-------------|------|---------------\n" +
 367         "`DefaultStackSize` | Overrides the default stack size for secondary threads | `STRING` | `0`\n" +
 368         "`DbgEnableMiniDump` | If set to 1, enables this core dump generation. The default is NOT to generate a dump | `DWORD` | `0`\n" +
 369         "`DbgMiniDumpName` | If set, use as the template to create the dump path and file name. The pid can be placed in the name with %d. | `STRING` | `_/tmp/coredump.%d_`\n" +
 370         "`DbgMiniDumpType` | If set to 1 generates _MiniDumpNormal_, 2 _MiniDumpWithPrivateReadWriteMemory_, 3 _MiniDumpFilterTriage_, 4 _MiniDumpWithFullMemory_ | `DWORD` | `1`\n" +
 371         "`CreateDumpDiagnostics` | If set to 1, enables the _createdump_ utilities diagnostic messages (TRACE macro) | `DWORD` | `0`\n";
 372
 373     /// <summary>
 374     /// Writes documentation file "clr-configuration-knobs.md"
 375     /// </summary>
 376     /// <param name="knobs"></param>
 377     /// <param name="filePath"></param>
 378     /// <returns></returns>
 379     public static string WriteFile(SortedDictionary<string, SortedDictionary<string, Knob>> knobs, string filePath = "clr-configuration-knobs.md")
 380     {
 381         int count = 0;
 382         var hashLong = GetRepoHeadHash();
 383         var date = DateTime.UtcNow.ToShortDateString();
 384
 385         using (StreamWriter writer = new StreamWriter(filePath, false, new UTF8Encoding(false)))
 386         {
 387             writer.NewLine = "\n";
 388
 389             writer.WriteLine(IntroSection);
 390
 391             writer.WriteLine(HostConfigurationKnobsPart1);
 392
 393             writer.WriteLine(HostConfigurationKnobsPart2);
 394
 395             writer.WriteLine(ClrConfigSectionHeader);
 396
 397             ClrConfigSectionInfo = ClrConfigSectionInfo.Replace("GIT_LONG_HASH", hashLong);
 398             ClrConfigSectionInfo = ClrConfigSectionInfo.Replace("GIT_SHORT_HASH", hashLong.Substring(0, 7));
 399             ClrConfigSectionInfo = ClrConfigSectionInfo.Replace("DATE_CREATED", date);
 400
 401             writer.WriteLine(ClrConfigSectionInfo);
 402
 403             writer.WriteLine(ClrConfigSectionUsage);
 404
 405             writer.WriteLine("#### Tables");
 406             int index = 1;
 407             foreach(string category in knobs.Keys)
 408             {
 409                 writer.WriteLine($"{index++}. [{category} Configuration Knobs](#{EscapeMdId(category)}-configuration-knobs)");
 410             }
 411
 412             writer.WriteLine();
 413
 414             foreach (string category in knobs.Keys)
 415             {
 416                 writer.WriteLine($"#### {category} Configuration Knobs");
 417                 writer.Write(ClrConfigTableHeader);
 418
 419                 var catKnobs = knobs[category];
 420
 421                 foreach (string key in catKnobs.Keys)
 422                 {
 423                     var knob = catKnobs[key];
 424                     writer.Write($"`{knob.Name}` | {knob.Description} | `{knob.Type}` | ");
 425                     writer.Write(knob.Class.Length > 0 ? $"`{knob.Class}` | " : " | ");
 426                     writer.Write(knob.DefaultValue.Length > 0 ? $"`{knob.DefaultValue}` | " : " | ");
 427                     writer.WriteLine($"{ knob.Flags}");
 428                     count++;
 429                 }
 430
 431                 writer.WriteLine();
 432             }
 433
 434             writer.WriteLine();
 435
 436             writer.WriteLine(PalConfigurationKnobs);
 437         }
 438         return $"Success: {count} parsed configuration knobs, documentation file {filePath} has been updated: commit {hashLong.Substring(0, 7)} date {date}.";
 439     }
 440
 441     private static string EscapeMdId(string value)
 442     {
 443         var buffer = value.ToLower().ToCharArray();
 444         StringBuilder sb = new StringBuilder(value.Length);
 445
 446         for (int i = 0; i < buffer.Length; i++)
 447         {
 448             switch (buffer[i])
 449             {
 450                 case '|':
 451                 case '(':
 452                 case ')':
 453                 case '{':
 454                 case '}':
 455                 case '[':
 456                 case ']':
 457                 case '?':
 458                 case '!':
 459                 case '\\':
 460                 case '/':
 461                     break;
 462
 463                 case ' ':
 464                     sb.Append('-');
 465                     break;
 466
 467                 default:
 468                     sb.Append(buffer[i]);
 469                     break;
 470             }
 471         }
 472
 473         return sb.ToString();
 474     }
 475
 476     public static string GetRepoHeadHash()
 477     {
 478         var startInfo = new ProcessStartInfo
 479         {
 480             FileName = "cmd.exe",
 481             Arguments = "/c \"git rev-parse HEAD\"",
 482             CreateNoWindow = true,
 483             RedirectStandardOutput = true,
 484             RedirectStandardError = true,
 485             UseShellExecute = false
 486         };
 487
 488         var process = Process.Start(startInfo);
 489
 490         if (process.WaitForExit(20000))
 491         {
 492             string result = process.StandardOutput.ReadToEnd();
 493             string error = process.StandardError.ReadToEnd();
 494             if (process.ExitCode == 0)
 495             {
 496                 return result.Trim();
 497             }
 498             else
 499             {
 500                 throw new InvalidOperationException(error);
 501             }
 502         }
 503         else
 504         {
 505             throw new TimeoutException("\"git rev-parse HEAD\" command timed out");
 506         }
 507     }
 508 }
 509
 510
 511 var knobsDictionary = new SortedDictionary<string, SortedDictionary<string, Knob>>(StringComparer.Create(CultureInfo.InvariantCulture, false));
 512
 513 WriteLine($"Processing header file {clrConfig}");
 514 ParseConfigFile(clrConfig, true, knobsDictionary);
 515
 516 WriteLine($"Processing header file {jitConfig}");
 517 ParseConfigFile(jitConfig, false, knobsDictionary);
 518
 519 WriteLine(ConfigKnobsDoc.WriteFile(knobsDictionary));