src/scripts/genEtwProvider.py

   1 ##
   2 ## Licensed to the .NET Foundation under one or more agreements.
   3 ## The .NET Foundation licenses this file to you under the MIT license.
   4 ## See the LICENSE file in the project root for more information.
   5 ##
   6 ## This script generates the interface to ETW using MC.exe
   7
   8 import os
   9 from os import path
  10 import shutil
  11 import re
  12 import sys
  13 import argparse
  14 import subprocess
  15 import xml.dom.minidom as DOM
  16 from genEventing import parseTemplateNodes
  17 from utilities import open_for_update, update_directory, parseExclusionList
  18
  19 macroheader_filename = "etwmacros.h"
  20 mcheader_filename = "ClrEtwAll.h"
  21 clrxplat_filename = "clrxplatevents.h"
  22 etw_dirname = "etw"
  23 replacements = [
  24     (r"EventEnabled", "EventXplatEnabled"),
  25     (r"\bPVOID\b", "void*"),
  26 ]
  27 counted_replacements = [
  28     # There is a bug in the MC code generator that miscomputes the size of ETW events
  29     # which have variable size arrays of variable size arrays. This occurred in our GCBulkType
  30     # event. This workaround replaces the bad size computation with the correct one. See
  31     # https://github.com/dotnet/coreclr/pull/25454 for more information"
  32     (r"_Arg0 \* _Arg2_Len_", "_Arg2_Len_", 1)
  33 ]
  34
  35 stdprolog_cpp="""
  36 // Licensed to the .NET Foundation under one or more agreements.
  37 // The .NET Foundation licenses this file to you under the MIT license.
  38 // See the LICENSE file in the project root for more information.
  39
  40 /******************************************************************
  41
  42 DO NOT MODIFY. AUTOGENERATED FILE.
  43 This file is generated using the logic from <root>/src/scripts/genEtwProvider.py
  44
  45 ******************************************************************/
  46 """
  47
  48 def genProviderInterface(manifest, intermediate):
  49     provider_dirname = os.path.join(intermediate, etw_dirname + "_temp")
  50
  51     if not os.path.exists(provider_dirname):
  52         os.makedirs(provider_dirname)
  53
  54     cmd = ['mc.exe', '-h', provider_dirname, '-r', provider_dirname, '-b', '-co', '-um', '-p', 'FireEtXplat', manifest]
  55     subprocess.check_call(cmd)
  56
  57     header_text = None
  58     with open(path.join(provider_dirname, mcheader_filename), 'r') as mcheader_file:
  59         header_text = mcheader_file.read()
  60
  61     for pattern, replacement in replacements:
  62         header_text = re.sub(pattern, replacement, header_text)
  63
  64     for pattern, replacement, expected_count in counted_replacements:
  65         (header_text, actual_count) = re.subn(pattern, replacement, header_text)
  66         if actual_count != expected_count:
  67             raise Exception("The workaround for https://github.com/dotnet/coreclr/pull/25454 in src/scripts/genEtwProvider.py could not be applied. Has the generated code changed or perhaps the underlying issue has been fixed? ")
  68
  69     with open(path.join(provider_dirname, mcheader_filename), 'w') as mcheader_file:
  70         mcheader_file.write(header_text)
  71
  72 def genXplatHeader(intermediate):
  73     with open_for_update(path.join(intermediate, clrxplat_filename)) as header_file:
  74         header_file.write("""
  75 #ifndef _CLR_XPLAT_EVENTS_H_
  76 #define _CLR_XPLAT_EVENTS_H_
  77
  78 #include "{0}/{1}"
  79 #include "{0}/{2}"
  80
  81 #endif //_CLR_XPLAT_EVENTS_H_
  82 """.format(etw_dirname, macroheader_filename, mcheader_filename))
  83
  84 def getStackWalkBit(eventProvider, taskName, eventSymbol, stackSet):
  85     for entry in stackSet:
  86         tokens = entry.split(':')
  87
  88         if len(tokens) != 3:
  89             raise Exception("Error, possible error in the script which introduced the entry "+ entry)
  90
  91         eventCond  = tokens[0] == eventProvider or tokens[0] == "*"
  92         taskCond   = tokens[1] == taskName      or tokens[1] == "*"
  93         symbolCond = tokens[2] == eventSymbol   or tokens[2] == "*"
  94
  95         if eventCond and taskCond and symbolCond:
  96             return False
  97     return True
  98
  99 #Add the miscellaneous checks here
 100 def checkConsistency(manifest, exclusion_filename):
 101     tree                      = DOM.parse(manifest)
 102     exclusionInfo = parseExclusionList(exclusion_filename)
 103     for providerNode in tree.getElementsByTagName('provider'):
 104
 105         stackSupportSpecified = {}
 106         eventNodes            = providerNode.getElementsByTagName('event')
 107         templateNodes         = providerNode.getElementsByTagName('template')
 108         eventProvider         = providerNode.getAttribute('name')
 109         allTemplates          = parseTemplateNodes(templateNodes)
 110
 111         for eventNode in eventNodes:
 112             taskName         = eventNode.getAttribute('task')
 113             eventSymbol      = eventNode.getAttribute('symbol')
 114             eventTemplate    = eventNode.getAttribute('template')
 115             eventValue       = int(eventNode.getAttribute('value'))
 116             clrInstanceBit   = getStackWalkBit(eventProvider, taskName, eventSymbol, exclusionInfo.noclrinstance)
 117             sLookupFieldName = "ClrInstanceID"
 118             sLookupFieldType = "win:UInt16"
 119
 120             if clrInstanceBit and allTemplates.get(eventTemplate):
 121                 # check for the event template and look for a field named ClrInstanceId of type win:UInt16
 122                 fnParam = allTemplates[eventTemplate].getFnParam(sLookupFieldName)
 123
 124                 if not(fnParam and fnParam.winType == sLookupFieldType):
 125                     raise Exception(exclusion_filename + ":No " + sLookupFieldName + " field of type " + sLookupFieldType + " for event symbol " +  eventSymbol)
 126
 127             # If some versions of an event are on the nostack/stack lists,
 128             # and some versions are not on either the nostack or stack list,
 129             # then developer likely forgot to specify one of the versions
 130
 131             eventStackBitFromNoStackList       = getStackWalkBit(eventProvider, taskName, eventSymbol, exclusionInfo.nostack)
 132             eventStackBitFromExplicitStackList = getStackWalkBit(eventProvider, taskName, eventSymbol, exclusionInfo.explicitstack)
 133             sStackSpecificityError = exclusion_filename + ": Error processing event :" + eventSymbol + "(ID" + str(eventValue) + "): This file must contain either ALL versions of this event or NO versions of this event. Currently some, but not all, versions of this event are present\n"
 134
 135             if not stackSupportSpecified.get(eventValue):
 136                  # Haven't checked this event before.  Remember whether a preference is stated
 137                 if ( not eventStackBitFromNoStackList) or ( not eventStackBitFromExplicitStackList):
 138                     stackSupportSpecified[eventValue] = True
 139                 else:
 140                     stackSupportSpecified[eventValue] = False
 141             else:
 142                 # We've checked this event before.
 143                 if stackSupportSpecified[eventValue]:
 144                     # When we last checked, a preference was previously specified, so it better be specified here
 145                     if eventStackBitFromNoStackList and eventStackBitFromExplicitStackList:
 146                         raise Exception(sStackSpecificityError)
 147                 else:
 148                     # When we last checked, a preference was not previously specified, so it better not be specified here
 149                     if ( not eventStackBitFromNoStackList) or ( not eventStackBitFromExplicitStackList):
 150                         raise Exception(sStackSpecificityError)
 151
 152 def genEtwMacroHeader(manifest, exclusion_filename, intermediate):
 153     provider_dirname = os.path.join(intermediate, etw_dirname + "_temp")
 154
 155     if not os.path.exists(provider_dirname):
 156         os.makedirs(provider_dirname)
 157
 158     tree                      = DOM.parse(manifest)
 159     numOfProviders            = len(tree.getElementsByTagName('provider'))
 160     nMaxEventBytesPerProvider = 64
 161
 162     exclusionInfo = parseExclusionList(exclusion_filename)
 163
 164     with open_for_update(os.path.join(provider_dirname, macroheader_filename)) as header_file:
 165         header_file.write(stdprolog_cpp + "\n")
 166
 167         header_file.write("#define NO_OF_ETW_PROVIDERS " + str(numOfProviders) + "\n")
 168         header_file.write("#define MAX_BYTES_PER_ETW_PROVIDER " + str(nMaxEventBytesPerProvider) + "\n")
 169         header_file.write("EXTERN_C SELECTANY const BYTE etwStackSupportedEvents[NO_OF_ETW_PROVIDERS][MAX_BYTES_PER_ETW_PROVIDER] = \n{\n")
 170
 171         for providerNode in tree.getElementsByTagName('provider'):
 172             stackSupportedEvents = [0]*nMaxEventBytesPerProvider
 173             eventNodes = providerNode.getElementsByTagName('event')
 174             eventProvider    = providerNode.getAttribute('name')
 175
 176             for eventNode in eventNodes:
 177                 taskName                = eventNode.getAttribute('task')
 178                 eventSymbol             = eventNode.getAttribute('symbol')
 179                 eventTemplate           = eventNode.getAttribute('template')
 180                 eventTemplate           = eventNode.getAttribute('template')
 181                 eventValue              = int(eventNode.getAttribute('value'))
 182                 eventIndex              = eventValue // 8
 183                 eventBitPositionInIndex = eventValue % 8
 184
 185                 eventStackBitFromNoStackList       = int(getStackWalkBit(eventProvider, taskName, eventSymbol, exclusionInfo.nostack))
 186                 eventStackBitFromExplicitStackList = int(getStackWalkBit(eventProvider, taskName, eventSymbol, exclusionInfo.explicitstack))
 187
 188                 # Shift those bits into position.  For the explicit stack list, swap 0 and 1, so the eventValue* variables
 189                 # have 1 in the position iff we should issue a stack for the event.
 190                 eventValueUsingNoStackListByPosition = (eventStackBitFromNoStackList << eventBitPositionInIndex)
 191                 eventValueUsingExplicitStackListByPosition = ((1 - eventStackBitFromExplicitStackList) << eventBitPositionInIndex)
 192
 193                 # Commit the values to the in-memory array that we'll dump into the header file
 194                 stackSupportedEvents[eventIndex] = stackSupportedEvents[eventIndex] | eventValueUsingNoStackListByPosition;
 195                 if eventStackBitFromExplicitStackList == 0:
 196                     stackSupportedEvents[eventIndex] = stackSupportedEvents[eventIndex] | eventValueUsingExplicitStackListByPosition
 197
 198             # print the bit array
 199             line = []
 200             line.append("\t{")
 201             for elem in stackSupportedEvents:
 202                 line.append(str(elem))
 203                 line.append(", ")
 204
 205             del line[-1]
 206             line.append("},")
 207             header_file.write(''.join(line) + "\n")
 208         header_file.write("};\n")
 209
 210 def genFiles(manifest, intermediate, exclusion_filename):
 211     if not os.path.exists(intermediate):
 212         os.makedirs(intermediate)
 213
 214     genProviderInterface(manifest, intermediate)
 215     genEtwMacroHeader(manifest, exclusion_filename, intermediate)
 216     genXplatHeader(intermediate)
 217
 218
 219 def main(argv):
 220     #parse the command line
 221     parser = argparse.ArgumentParser(description="Generates the Code required to instrument ETW logging mechanism")
 222
 223     required = parser.add_argument_group('required arguments')
 224     required.add_argument('--man',  type=str, required=True,
 225                                     help='full path to manifest containig the description of events')
 226     required.add_argument('--exc',  type=str, required=True,
 227                                     help='full path to exclusion list')
 228     required.add_argument('--intermediate', type=str, required=True,
 229                                     help='full path to eventprovider  intermediate directory')
 230     args, unknown = parser.parse_known_args(argv)
 231     if unknown:
 232         print('Unknown argument(s): ', ', '.join(unknown))
 233         return 1
 234
 235     manifest           = args.man
 236     exclusion_filename = args.exc
 237     intermediate       = args.intermediate
 238
 239     checkConsistency(manifest, exclusion_filename)
 240     genFiles(manifest, intermediate, exclusion_filename)
 241
 242     # Update the final directory from temp
 243     provider_temp_dirname = os.path.join(intermediate, etw_dirname + "_temp")
 244     provider_dirname = os.path.join(intermediate, etw_dirname)
 245     if not os.path.exists(provider_dirname):
 246         os.makedirs(provider_dirname)
 247
 248     update_directory(provider_temp_dirname, provider_dirname)
 249     shutil.rmtree(provider_temp_dirname)
 250
 251 if __name__ == '__main__':
 252     return_code = main(sys.argv[1:])
 253     sys.exit(return_code)