From: Lluís Vilanova Date: Tue, 3 Apr 2012 18:47:39 +0000 (+0200) Subject: tracetool: Rewrite infrastructure as python modules X-Git-Tag: TizenStudio_2.0_p2.3.2~208^2~4227^2~8 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=650ab98d1d9551f0ca2180c0d88427acfcb081cf;p=sdk%2Femulator%2Fqemu.git tracetool: Rewrite infrastructure as python modules The tracetool script is written in shell and has hit several portability problems due to shell quirks or external tools across host platforms. Additionally the amount of string processing and lack of real data structures makes it tough to implement code generator backends for tracers that are more complex. This patch replaces the shell version of tracetool with a Python version. The new tracetool design is: scripts/tracetool.py - top-level script scripts/tracetool/backend/ - tracer backends live here (simple, ust) scripts/tracetool/format/ - output formats live here (.c, .h) There is common code for trace-events definition parsing so that backends can focus on generating code rather than parsing input. Support for all existing backends (nop, stderr, simple, ust, and dtrace) is added back in follow-up patches. [Commit description written by Stefan Hajnoczi] Signed-off-by: Lluís Vilanova Signed-off-by: Stefan Hajnoczi --- diff --git a/Makefile.objs b/Makefile.objs index 5c3bcda..6d6f24d 100644 --- a/Makefile.objs +++ b/Makefile.objs @@ -378,12 +378,12 @@ else trace.h: trace.h-timestamp endif trace.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak - $(call quiet-command,sh $(SRC_PATH)/scripts/tracetool --$(TRACE_BACKEND) -h < $< > $@," GEN trace.h") + $(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/tracetool.py --format=h --backend=$(TRACE_BACKEND) < $< > $@," GEN trace.h") @cmp -s $@ trace.h || cp $@ trace.h trace.c: trace.c-timestamp trace.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak - $(call quiet-command,sh $(SRC_PATH)/scripts/tracetool --$(TRACE_BACKEND) -c < $< > $@," GEN trace.c") + $(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/tracetool.py --format=c --backend=$(TRACE_BACKEND) < $< > $@," GEN trace.c") @cmp -s $@ trace.c || cp $@ trace.c trace.o: trace.c $(GENERATED_HEADERS) @@ -396,7 +396,7 @@ trace-dtrace.h: trace-dtrace.dtrace # rule file. So we use '.dtrace' instead trace-dtrace.dtrace: trace-dtrace.dtrace-timestamp trace-dtrace.dtrace-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak - $(call quiet-command,sh $(SRC_PATH)/scripts/tracetool --$(TRACE_BACKEND) -d < $< > $@," GEN trace-dtrace.dtrace") + $(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/tracetool.py --format=d --backend=$(TRACE_BACKEND) < $< > $@," GEN trace-dtrace.dtrace") @cmp -s $@ trace-dtrace.dtrace || cp $@ trace-dtrace.dtrace trace-dtrace.o: trace-dtrace.dtrace $(GENERATED_HEADERS) diff --git a/Makefile.target b/Makefile.target index 84951a0..a0540cd 100644 --- a/Makefile.target +++ b/Makefile.target @@ -59,12 +59,13 @@ TARGET_TYPE=system endif $(QEMU_PROG).stp: $(SRC_PATH)/trace-events - $(call quiet-command,sh $(SRC_PATH)/scripts/tracetool \ - --$(TRACE_BACKEND) \ - --binary $(bindir)/$(QEMU_PROG) \ - --target-arch $(TARGET_ARCH) \ - --target-type $(TARGET_TYPE) \ - --stap < $(SRC_PATH)/trace-events > $(QEMU_PROG).stp," GEN $(QEMU_PROG).stp") + $(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/tracetool.py \ + --format=stap \ + --backend=$(TRACE_BACKEND) \ + --binary=$(bindir)/$(QEMU_PROG) \ + --target-arch=$(TARGET_ARCH) \ + --target-type=$(TARGET_TYPE) \ + < $(SRC_PATH)/trace-events > $(QEMU_PROG).stp," GEN $(QEMU_PROG).stp") else stap: endif diff --git a/configure b/configure index 2d62d12..03b49f6 100755 --- a/configure +++ b/configure @@ -1097,7 +1097,7 @@ echo " --disable-docs disable documentation build" echo " --disable-vhost-net disable vhost-net acceleration support" echo " --enable-vhost-net enable vhost-net acceleration support" echo " --enable-trace-backend=B Set trace backend" -echo " Available backends:" $("$source_path"/scripts/tracetool --list-backends) +echo " Available backends:" $($python "$source_path"/scripts/tracetool.py --list-backends) echo " --with-trace-file=NAME Full PATH,NAME of file to store traces" echo " Default:trace-" echo " --disable-spice disable spice" @@ -2670,7 +2670,7 @@ fi ########################################## # check if trace backend exists -sh "$source_path/scripts/tracetool" "--$trace_backend" --check-backend > /dev/null 2> /dev/null +$python "$source_path/scripts/tracetool.py" "--backend=$trace_backend" --check-backend > /dev/null 2> /dev/null if test "$?" -ne 0 ; then echo echo "Error: invalid trace backend" diff --git a/scripts/tracetool b/scripts/tracetool deleted file mode 100755 index 7b1c142..0000000 --- a/scripts/tracetool +++ /dev/null @@ -1,666 +0,0 @@ -#!/bin/sh -# -# Code generator for trace events -# -# Copyright IBM, Corp. 2010 -# -# This work is licensed under the terms of the GNU GPL, version 2. See -# the COPYING file in the top-level directory. - -# Disable pathname expansion, makes processing text with '*' characters simpler -set -f - -usage() -{ - cat >&2 < return 0 if property is present, or 1 otherwise -has_property() -{ - local props prop - props=${1%%\(*} - props=${props% *} - for prop in $props; do - if [ "$prop" = "$2" ]; then - return 0 - fi - done - return 1 -} - -# Get the argument list of a trace event, including types and names -get_args() -{ - local args - args=${1#*\(} - args=${args%%\)*} - echo "$args" - - if (echo "$args" | grep "[ *]next\($\|[, ]\)" > /dev/null 2>&1); then - echo -e "\n#error 'next' is a bad argument name (clash with systemtap keyword)\n " - fi -} - -# Get the argument name list of a trace event -get_argnames() -{ - local nfields field name sep - nfields=0 - sep="$2" - for field in $(get_args "$1"); do - nfields=$((nfields + 1)) - - # Drop pointer star - field=${field#\*} - - # Only argument names have commas at the end - name=${field%,} - test "$field" = "$name" && continue - - printf "%s%s " $name $sep - done - - # Last argument name - if [ "$nfields" -gt 1 ] - then - printf "%s" "$name" - fi -} - -# Get the number of arguments to a trace event -get_argc() -{ - local name argc - argc=0 - for name in $(get_argnames "$1", ","); do - argc=$((argc + 1)) - done - echo $argc -} - -# Get the format string including double quotes for a trace event -get_fmt() -{ - puts "${1#*)}" -} - -linetoh_begin_nop() -{ - return -} - -linetoh_nop() -{ - local name args - name=$(get_name "$1") - args=$(get_args "$1") - - # Define an empty function for the trace event - cat < -#include "trace/stderr.h" - -extern TraceEvent trace_list[]; -EOF - - stderr_event_num=0 -} - -linetoh_stderr() -{ - local name args argnames argc fmt - name=$(get_name "$1") - args=$(get_args "$1") - argnames=$(get_argnames "$1" ",") - argc=$(get_argc "$1") - fmt=$(get_fmt "$1") - - if [ "$argc" -gt 0 ]; then - argnames=", $argnames" - fi - - cat <" - ust_clean_namespace -} - -linetoh_ust() -{ - local name args argnames - name=$(get_name "$1") - args=$(get_args "$1") - argnames=$(get_argnames "$1", ",") - - cat < -$(ust_clean_namespace) -#include "trace.h" -EOF -} - -linetoc_ust() -{ - local name args argnames fmt - name=$(get_name "$1") - args=$(get_args "$1") - argnames=$(get_argnames "$1", ",") - [ -z "$argnames" ] || argnames=", $argnames" - fmt=$(get_fmt "$1") - - cat < --backend= [] + +Backends: +%(backends)s + +Formats: +%(formats)s + +Options: + --help This help message. + --list-backends Print list of available backends. + --check-backend Check if the given backend is valid. +""" % { + "script" : _SCRIPT, + "backends" : backend_descr, + "formats" : format_descr, + }) + + if msg is None: + sys.exit(0) + else: + sys.exit(1) + + +def main(args): + global _SCRIPT + _SCRIPT = args[0] + + long_opts = [ "backend=", "format=", "help", "list-backends", "check-backend" ] + long_opts += [ "binary=", "target-type=", "target-arch=", "probe-prefix=" ] + + try: + opts, args = getopt.getopt(args[1:], "", long_opts) + except getopt.GetoptError as err: + error_opt(str(err)) + + check_backend = False + arg_backend = "" + arg_format = "" + for opt, arg in opts: + if opt == "--help": + error_opt() + + elif opt == "--backend": + arg_backend = arg + elif opt == "--format": + arg_format = arg + + elif opt == "--list-backends": + backends = tracetool.backend.get_list() + out(", ".join([ b for b,_ in backends ])) + sys.exit(0) + elif opt == "--check-backend": + check_backend = True + + else: + error_opt("unhandled option: %s" % opt) + + if arg_backend is None: + error_opt("backend not set") + + if check_backend: + if tracetool.backend.exists(arg_backend): + sys.exit(0) + else: + sys.exit(1) + + try: + tracetool.generate(sys.stdin, arg_format, arg_backend) + except tracetool.TracetoolError as e: + error_opt(str(e)) + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/tracetool/__init__.py b/scripts/tracetool/__init__.py new file mode 100644 index 0000000..1719bb4 --- /dev/null +++ b/scripts/tracetool/__init__.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Machinery for generating tracing-related intermediate files. +""" + +__author__ = "Lluís Vilanova " +__copyright__ = "Copyright 2012, Lluís Vilanova " +__license__ = "GPL version 2 or (at your option) any later version" + +__maintainer__ = "Stefan Hajnoczi" +__email__ = "stefanha@linux.vnet.ibm.com" + + +import re +import sys + +import tracetool.format +import tracetool.backend + + +def error_write(*lines): + """Write a set of error lines.""" + sys.stderr.writelines("\n".join(lines) + "\n") + +def error(*lines): + """Write a set of error lines and exit.""" + error_write(*lines) + sys.exit(1) + + +def out(*lines, **kwargs): + """Write a set of output lines. + + You can use kwargs as a shorthand for mapping variables when formating all + the strings in lines. + """ + lines = [ l % kwargs for l in lines ] + sys.stdout.writelines("\n".join(lines) + "\n") + + +class Arguments: + """Event arguments description.""" + + def __init__(self, args): + """ + Parameters + ---------- + args : + List of (type, name) tuples. + """ + self._args = args + + @staticmethod + def build(arg_str): + """Build and Arguments instance from an argument string. + + Parameters + ---------- + arg_str : str + String describing the event arguments. + """ + res = [] + for arg in arg_str.split(","): + arg = arg.strip() + parts = arg.split() + head, sep, tail = parts[-1].rpartition("*") + parts = parts[:-1] + if tail == "void": + assert len(parts) == 0 and sep == "" + continue + arg_type = " ".join(parts + [ " ".join([head, sep]).strip() ]).strip() + res.append((arg_type, tail)) + return Arguments(res) + + def __iter__(self): + """Iterate over the (type, name) pairs.""" + return iter(self._args) + + def __len__(self): + """Number of arguments.""" + return len(self._args) + + def __str__(self): + """String suitable for declaring function arguments.""" + if len(self._args) == 0: + return "void" + else: + return ", ".join([ " ".join([t, n]) for t,n in self._args ]) + + def __repr__(self): + """Evaluable string representation for this object.""" + return "Arguments(\"%s\")" % str(self) + + def names(self): + """List of argument names.""" + return [ name for _, name in self._args ] + + def types(self): + """List of argument types.""" + return [ type_ for type_, _ in self._args ] + + +class Event(object): + """Event description. + + Attributes + ---------- + name : str + The event name. + fmt : str + The event format string. + properties : set(str) + Properties of the event. + args : Arguments + The event arguments. + """ + + _CRE = re.compile("((?P.*)\s+)?(?P[^(\s]+)\((?P[^)]*)\)\s*(?P\".*)?") + + _VALID_PROPS = set(["disable"]) + + def __init__(self, name, props, fmt, args): + """ + Parameters + ---------- + name : string + Event name. + props : list of str + Property names. + fmt : str + Event printing format. + args : Arguments + Event arguments. + """ + self.name = name + self.properties = props + self.fmt = fmt + self.args = args + + unknown_props = set(self.properties) - self._VALID_PROPS + if len(unknown_props) > 0: + raise ValueError("Unknown properties: %s" % ", ".join(unknown_props)) + + @staticmethod + def build(line_str): + """Build an Event instance from a string. + + Parameters + ---------- + line_str : str + Line describing the event. + """ + m = Event._CRE.match(line_str) + assert m is not None + groups = m.groupdict('') + + name = groups["name"] + props = groups["props"].split() + fmt = groups["fmt"] + args = Arguments.build(groups["args"]) + + return Event(name, props, fmt, args) + + def __repr__(self): + """Evaluable string representation for this object.""" + return "Event('%s %s(%s) %s')" % (" ".join(self.properties), + self.name, + self.args, + self.fmt) + +def _read_events(fobj): + res = [] + for line in fobj: + if not line.strip(): + continue + if line.lstrip().startswith('#'): + continue + res.append(Event.build(line)) + return res + + +class TracetoolError (Exception): + """Exception for calls to generate.""" + pass + + +def try_import(mod_name, attr_name = None, attr_default = None): + """Try to import a module and get an attribute from it. + + Parameters + ---------- + mod_name : str + Module name. + attr_name : str, optional + Name of an attribute in the module. + attr_default : optional + Default value if the attribute does not exist in the module. + + Returns + ------- + A pair indicating whether the module could be imported and the module or + object or attribute value. + """ + try: + module = __import__(mod_name, fromlist=["__package__"]) + if attr_name is None: + return True, module + return True, getattr(module, str(attr_name), attr_default) + except ImportError: + return False, None + + +def generate(fevents, format, backend): + """Generate the output for the given (format, backend) pair. + + Parameters + ---------- + fevents : file + Event description file. + format : str + Output format name. + backend : str + Output backend name. + """ + # fix strange python error (UnboundLocalError tracetool) + import tracetool + + format = str(format) + if len(format) is 0: + raise TracetoolError("format not set") + mformat = format.replace("-", "_") + if not tracetool.format.exists(mformat): + raise TracetoolError("unknown format: %s" % format) + + backend = str(backend) + if len(backend) is 0: + raise TracetoolError("backend not set") + mbackend = backend.replace("-", "_") + if not tracetool.backend.exists(mbackend): + raise TracetoolError("unknown backend: %s" % backend) + + if not tracetool.backend.compatible(mbackend, mformat): + raise TracetoolError("backend '%s' not compatible with format '%s'" % + (backend, format)) + + events = _read_events(fevents) + + if backend == "nop": + ( e.properies.add("disable") for e in events ) + + tracetool.format.generate_begin(mformat, events) + tracetool.backend.generate("nop", format, + [ e + for e in events + if "disable" in e.properties ]) + tracetool.backend.generate(backend, format, + [ e + for e in events + if "disable" not in e.properties ]) + tracetool.format.generate_end(mformat, events) diff --git a/scripts/tracetool/backend/__init__.py b/scripts/tracetool/backend/__init__.py new file mode 100644 index 0000000..34b7ed8 --- /dev/null +++ b/scripts/tracetool/backend/__init__.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Backend management. + + +Creating new backends +--------------------- + +A new backend named 'foo-bar' corresponds to Python module +'tracetool/backend/foo_bar.py'. + +A backend module should provide a docstring, whose first non-empty line will be +considered its short description. + +All backends must generate their contents through the 'tracetool.out' routine. + + +Backend functions +----------------- + +======== ======================================================================= +Function Description +======== ======================================================================= + Called to generate the format- and backend-specific code for each of + the specified events. If the function does not exist, the backend is + considered not compatible with the given format. +======== ======================================================================= +""" + +__author__ = "Lluís Vilanova " +__copyright__ = "Copyright 2012, Lluís Vilanova " +__license__ = "GPL version 2 or (at your option) any later version" + +__maintainer__ = "Stefan Hajnoczi" +__email__ = "stefanha@linux.vnet.ibm.com" + + +import pkgutil + +import tracetool + + +def get_list(): + """Get a list of (name, description) pairs.""" + res = [("nop", "Tracing disabled.")] + for _, modname, _ in pkgutil.iter_modules(tracetool.backend.__path__): + module = tracetool.try_import("tracetool.backend." + modname) + + # just in case; should never fail unless non-module files are put there + if not module[0]: + continue + module = module[1] + + doc = module.__doc__ + if doc is None: + doc = "" + doc = doc.strip().split("\n")[0] + + name = modname.replace("_", "-") + res.append((name, doc)) + return res + + +def exists(name): + """Return whether the given backend exists.""" + if len(name) == 0: + return False + if name == "nop": + return True + name = name.replace("-", "_") + return tracetool.try_import("tracetool.backend." + name)[1] + + +def compatible(backend, format): + """Whether a backend is compatible with the given format.""" + if not exists(backend): + raise ValueError("unknown backend: %s" % backend) + + backend = backend.replace("-", "_") + format = format.replace("-", "_") + + if backend == "nop": + return True + else: + func = tracetool.try_import("tracetool.backend." + backend, + format, None)[1] + return func is not None + + +def _empty(events): + pass + +def generate(backend, format, events): + """Generate the per-event output for the given (backend, format) pair.""" + if not compatible(backend, format): + raise ValueError("backend '%s' not compatible with format '%s'" % + (backend, format)) + + backend = backend.replace("-", "_") + format = format.replace("-", "_") + + if backend == "nop": + func = tracetool.try_import("tracetool.format." + format, + "nop", _empty)[1] + else: + func = tracetool.try_import("tracetool.backend." + backend, + format, None)[1] + + func(events) diff --git a/scripts/tracetool/format/__init__.py b/scripts/tracetool/format/__init__.py new file mode 100644 index 0000000..0e4baf0 --- /dev/null +++ b/scripts/tracetool/format/__init__.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Format management. + + +Creating new formats +-------------------- + +A new format named 'foo-bar' corresponds to Python module +'tracetool/format/foo_bar.py'. + +A format module should provide a docstring, whose first non-empty line will be +considered its short description. + +All formats must generate their contents through the 'tracetool.out' routine. + + +Format functions +---------------- + +All the following functions are optional, and no output will be generated if +they do not exist. + +======== ======================================================================= +Function Description +======== ======================================================================= +begin Called to generate the format-specific file header. +end Called to generate the format-specific file footer. +nop Called to generate the per-event contents when the event is disabled or + the selected backend is 'nop'. +======== ======================================================================= +""" + +__author__ = "Lluís Vilanova " +__copyright__ = "Copyright 2012, Lluís Vilanova " +__license__ = "GPL version 2 or (at your option) any later version" + +__maintainer__ = "Stefan Hajnoczi" +__email__ = "stefanha@linux.vnet.ibm.com" + + +import pkgutil + +import tracetool + + +def get_list(): + """Get a list of (name, description) pairs.""" + res = [] + for _, modname, _ in pkgutil.iter_modules(tracetool.format.__path__): + module = tracetool.try_import("tracetool.format." + modname) + + # just in case; should never fail unless non-module files are put there + if not module[0]: + continue + module = module[1] + + doc = module.__doc__ + if doc is None: + doc = "" + doc = doc.strip().split("\n")[0] + + name = modname.replace("_", "-") + res.append((name, doc)) + return res + + +def exists(name): + """Return whether the given format exists.""" + if len(name) == 0: + return False + name = name.replace("-", "_") + return tracetool.try_import("tracetool.format." + name)[1] + + +def _empty(events): + pass + +def generate_begin(name, events): + """Generate the header of the format-specific file.""" + if not exists(name): + raise ValueError("unknown format: %s" % name) + + name = name.replace("-", "_") + func = tracetool.try_import("tracetool.format." + name, + "begin", _empty)[1] + func(events) + +def generate_end(name, events): + """Generate the footer of the format-specific file.""" + if not exists(name): + raise ValueError("unknown format: %s" % name) + + name = name.replace("-", "_") + func = tracetool.try_import("tracetool.format." + name, + "end", _empty)[1] + func(events)