Parse usbmon tcpdump logs directly
authorMauro Carvalho Chehab <mchehab@redhat.com>
Tue, 8 Mar 2011 15:59:53 +0000 (12:59 -0300)
committerMauro Carvalho Chehab <mchehab@redhat.com>
Tue, 8 Mar 2011 16:26:06 +0000 (13:26 -0300)
Instead of relying on wireshark parser, just read the tcpdump
logs directly, and use it to produce a parseable output.

I started to use my Beagleboard as an USB sniffer using the
code at:
http://beagleboard-usbsniffer.blogspot.com/

However, as I want to use the parser scripts for USB that I wrote,
I decided to write a parser that outputs data on a similar format.

On a first trial, I tried to parse the wireshark dump, but this
didn't work fine. So, I rewrote my parser to get the tcpdump
directly, using the cpan Net::TcpDumpLog.

There are some adjustments probably needed, but this version
already produces something usefull.

Signed-off-by: Mauro Carvalho Chehab <mchehab@redhat.com>
contrib/parse_tcpdump_log.pl [new file with mode: 0755]
contrib/wireshark_parser.pl [deleted file]

diff --git a/contrib/parse_tcpdump_log.pl b/contrib/parse_tcpdump_log.pl
new file mode 100755 (executable)
index 0000000..ffb23f0
--- /dev/null
@@ -0,0 +1,232 @@
+#!/usr/bin/perl
+# using cpan, you should install Net::TcpDumpLog
+use Net::TcpDumpLog;
+use strict;
+
+
+# Currently, accepts only one usbmon format:
+#      USB with padded Linux header (LINKTYPE_USB_LINUX_MMAPPED)
+# This is the one produced by Beagleboard sniffer GSOC.
+
+my $debug = 0;
+
+# Frame format as parsed by libpcap 1.0.0 and 1.1.1. Not sure if format
+# changed on different versions.
+
+my $filename;
+
+# FIXME: use shift of die, after finishing the tests
+$filename = shift or die "Please specify a file name";
+
+my @pending;
+
+my $initial_time;
+my $last_time;
+
+sub print_frame($$)
+{
+       my %req = %{ @_[0] };
+       my %resp = %{ @_[1] };
+
+       # Print timestamps:
+       #       relative time from resp 1
+       #       relative time from last resp
+       #       time to complete
+       printf "%09d ms %06d ms (%06d us",
+               1000 * $req{"Time"},
+               1000 * ($req{"Time"} - $last_time),
+               ($resp{"Time"} - $req{"Time"}) * 1000000;
+       $last_time = $req{"Time"};
+
+       printf " EP=%02x)", $resp{"Endpoint"};
+
+       my $app_data = substr($req{"Payload"}, 0, 8 * 2);
+       my $type = hex(substr($app_data, 0, 2));
+       while ($app_data ne "") {
+               printf " %s", substr($app_data, 0, 2);
+               $app_data = substr($app_data, 2);
+       }
+
+       # Extra data
+
+       if ($type > 128) {
+               printf " <<<";
+       } else {
+               printf " >>>";
+       }
+
+       my $app_data = substr($req{"Payload"}, 24 * 2);
+       while ($app_data ne "") {
+               printf " %02x", substr($app_data, 0, 2);
+               $app_data = substr($app_data, 2);
+       }
+
+       my $app_data = substr($resp{"Payload"}, 24 * 2);
+       while ($app_data ne "") {
+               printf " %02x", substr($app_data, 0, 2);
+               $app_data = substr($app_data, 2);
+       }
+
+       print "\n";
+
+       if ($debug) {
+               my ($key, $value);
+               print "\tREQ:  $key => $value\n" while (($key, $value) = each(%req));
+               print "\tRESP: $key => $value\n" while (($key, $value) = each(%resp));
+               print "\n";
+       }
+
+       return;
+}
+
+sub process_frame($) {
+       my %frame = %{ @_[0] };
+
+       $initial_time = $frame{"Arrival"} if (!$initial_time);
+
+       if ($debug > 1) {
+               my ($key, $value);
+               print "\t\tRAW: $key => $value\n" while (($key, $value) = each(%frame));
+               print "\n";
+       }
+
+       # For now, we'll take a look only on control frames
+       return if ($frame{"TransferType"} ne "2");
+
+       if ($frame{"Status"} eq "-115") {
+               push @pending, \%frame;
+               return;
+       }
+
+       # Seek for operation origin
+       my $related = $frame{"ID"};
+       if (!$related) {
+               print "URB %d incomplete\n", $frame{"ID"};
+               return;
+       }
+       for (my $i = 0; $i < scalar(@pending); $i++) {
+               if ($related == $pending[$i]{"ID"}) {
+                       my %req = %{$pending[$i]};
+
+                       print_frame (\%req, \%frame);
+
+                       # Remove from array, as it were already used
+                       splice(@pending, $i, 1);
+                       return;
+               }
+       }
+       printf "URB %d incomplete: Couldn't find related URB %d\n", $related;
+       return;
+}
+
+sub parse_file($)
+{
+       my $file = shift;
+
+       my $log = Net::TcpDumpLog->new();
+       $log->read($file);
+
+       # Check for LINKTYPE_USB_LINUX_MMAPPED (220)
+       if ($log->linktype() != 220) {
+               printf"Link type %d\n", $log->linktype();
+               die "Link type is not USB";
+       }
+       my @Indexes = $log->indexes;
+
+       foreach my $index (@Indexes) {
+               my %frame;
+               my ($length_orig,$length_incl,$drops,$secs,$msecs) = $log->header($index);
+               $frame{"Time"} = sprintf "%d.%06d", $secs,$msecs;
+
+               my $strdata = $log->data($index);
+               my @data=unpack('C*', $strdata);
+
+               if ($debug > 2) {
+                       for (my $i = 0; $i < scalar(@data); $i++) {
+                               printf " %02x", $data[$i];
+                       }
+                       print "\n";
+               }
+
+       #typedef struct _usb_header_mmapped {
+       #       u_int64_t id;
+       #       u_int8_t event_type;
+       #       u_int8_t transfer_type;
+       #       u_int8_t endpoint_number;
+       #       u_int8_t device_address;
+       #       u_int16_t bus_id;
+       #       char setup_flag;/*if !=0 the urb setup header is not present*/
+       #       char data_flag; /*if !=0 no urb data is present*/
+       #       int64_t ts_sec;
+       #       int32_t ts_usec;
+       #       int32_t status;
+       #       u_int32_t urb_len;
+       #       u_int32_t data_len; /* amount of urb data really present in this event*/
+       #       union {
+       #               pcap_usb_setup setup;
+       #               iso_rec iso;
+       #       } s;
+       #       int32_t interval;       /* for Interrupt and Isochronous events */
+       #       int32_t start_frame;    /* for Isochronous events */
+       #       u_int32_t xfer_flags;   /* copy of URB's transfer flags */
+       #       u_int32_t ndesc;        /* number of isochronous descriptors */
+       #} pcap_usb_header_mmapped;
+
+               # Not sure if this would work on 32-bits machines
+               $frame{"ID"} = $data[0]       | $data[1] << 8 |
+                       $data[2] << 16 | $data[3] << 24 |
+                       $data[4] << 32 | $data[5] << 40 |
+                       $data[6] << 48 | $data[7] << 56;
+               $frame{"Type"} = chr($data[8]);
+               $frame{"TransferType"} = $data[9];
+               $frame{"Endpoint"} = $data[10];
+               $frame{"Device"} = $data[11];
+               $frame{"BusID"} = $data[12] | $data[13] << 8;
+               if ($data[14] == 0) {
+                       $frame{"SetupRequest"} = "present";
+               } else {
+                       $frame{"SetupRequest"} = "not present";
+               }
+               if ($data[15] == 0) {
+                       $frame{"HasData"} = "present";
+               } else {
+                       $frame{"HasData"} = "not present";
+               }
+               my $tsSec = $data[16]       | $data[17] << 8 |
+                       $data[18] << 16 | $data[19] << 24 |
+                       $data[20] << 32 | $data[21] << 40 |
+                       $data[22] << 48 | $data[23] << 56;
+               my $tsUsec = $data[24]       | $data[25] << 8 |
+                       $data[26] << 16 | $data[27] << 24;
+               $frame{"ArrivalTime"} = sprintf "%d.%06d", $tsSec,$tsUsec;
+
+               # Status is signed with 32 bits. Fix signal, as errors are negative
+               $frame{"Status"} = $data[28]       | $data[29] << 8 |
+                               $data[30] << 16 | $data[31] << 24;
+               $frame{"Status"} = $frame{"Status"} - 0x100000000 if ($frame{"Status"} & 0x80000000);
+
+               $frame{"URBLength"} = $data[32]       | $data[33] << 8 |
+                               $data[34] << 16 | $data[35] << 24;
+               $frame{"DataLength"} = $data[36]       | $data[37] << 8 |
+                               $data[38] << 16 | $data[39] << 24;
+
+               my $payload;
+               my $payload_size;
+               for (my $i = 40; $i < scalar(@data); $i++) {
+                       $payload .= sprintf "%02x", $data[$i];
+                       $payload_size++;
+               }
+               $frame{"Payload"} = $payload;
+               $frame{"PayloadSize"} = $payload_size;
+
+               if ($debug > 1) {
+                       my ($key, $value);
+                       print "\t$key => $value\n" while (($key, $value) = each(%frame));
+                       printf "\n";
+               }
+               process_frame(\%frame);
+       }
+}
+
+# Main program
+parse_file $filename;
\ No newline at end of file
diff --git a/contrib/wireshark_parser.pl b/contrib/wireshark_parser.pl
deleted file mode 100755 (executable)
index 1c74ae1..0000000
+++ /dev/null
@@ -1,327 +0,0 @@
-#!/usr/bin/perl
-use strict;
-use Date::Parse;
-
-my $debug = 1;
-
-my @pending;
-
-my $initial_time;
-my $last_time;
-
-sub print_frame($$)
-{
-       my %req = %{ @_[0] };
-       my %resp = %{ @_[1] };
-
-#      # For now, let's concern only when there are some data
-#      return if (!$resp{"ApplicationData"});
-
-       my $rel_time = $req{"Arrival"} - $initial_time;
-
-       # Print timestamps:
-       #       relative time from resp 1
-       #       relative time from last resp
-       #       time to complete
-       printf "%09d ms %06d ms (%06d us",
-               1000 * $req{"Time"},
-               1000 * ($req{"Time"} - $last_time),
-               ($resp{"Time"} - $req{"Time"}) * 1000000;
-       $last_time = $req{"Time"};
-
-       printf " EP=%s)", $resp{"Endpoint"};
-
-       printf " %02x", $req{"bmRequestType"};
-       printf " %02x", $req{"bRequest"} if ($req{"bRequest"});
-       printf " %02x", $req{"Index"} if ($req{"Index"});
-       printf " %02x", $req{"bDescriptorType"} if ($req{"bDescriptorType"});
-       printf " %02x %02x", $req{"LanguageId"} & 0xff, $req{"LanguageId"} >> 8 if ($req{"LanguageId"});
-       printf " %02x %02x", $req{"wLength"} & 0xff, $req{"wLength"} >> 8  if ($req{"wLength"});
-
-       my $app_data = $req{"ApplicationData"};
-       if ($app_data ne "") {
-               printf " >>>";
-       }
-       while ($app_data ne "") {
-               printf " %s", substr($app_data, 0, 2);
-               $app_data = substr ($app_data, 2);
-       }
-
-       my $app_data = $resp{"ApplicationData"};
-       if ($app_data ne "") {
-               printf " <<<";
-       }
-       while ($app_data ne "") {
-               printf " %s", substr($app_data, 0, 2);
-               $app_data = substr ($app_data, 2);
-       }
-
-
-       print "\n";
-
-       if ($debug) {
-               my ($key, $value);
-               print "\tREQ:  $key => $value\n" while (($key, $value) = each(%req));
-               print "\tRESP: $key => $value\n" while (($key, $value) = each(%resp));
-               print "\n";
-       }
-
-       return;
-}
-
-sub process_frame(%) {
-       my %frame = @_;
-
-       $initial_time = $frame{"Arrival"} if (!$initial_time);
-
-       if ($debug > 1) {
-               my ($key, $value);
-               print "\t\tRAW: $key => $value\n" while (($key, $value) = each(%frame));
-               print "\n";
-       }
-
-       # For now, we'll take a look only on control frames
-       return if ($frame{"TransferType"} ne "URB_CONTROL");
-
-       if ($frame{"Status"} eq "-EINPROGRESS") {
-               push @pending, \%frame;
-               return;
-       }
-
-       # Seek for operation origin
-       my $related = $frame{"__RelatedTo"};
-       if (!$related) {
-               print "URB %d incomplete\n", $frame{"Number"};
-               return;
-       }
-       for (my $i = 0; $i < scalar(@pending); $i++) {
-               if ($related == $pending[$i]{"Number"}) {
-                       my %req = %{$pending[$i]};
-
-                       print_frame (\%req, \%frame);
-
-                       # Remove from array, as it were already used
-                       splice(@pending, $i, 1);
-                       return;
-               }
-       }
-       printf "URB %d incomplete: Couldn't find related URB %d\n", $frame{"Number"}, $related;
-       return;
-}
-
-sub wireshark_parser() {
-       my %frame;
-       my $next_is_time_frame;
-
-       while (<>) {
-               next if (m/^\n/);
-               next if (m/^\s+(INTERFACE|ENDPOINT|DEVICE|CONFIGURATION|STRING)\s+DESCRIPTOR/);
-               next if (m/^\s+\[Protocols\s+in\s+frame:\s+usb\]/);
-               if (m/^No.\s+Time\s+Source\s+Destination\s+Protocol Info/) {
-                       process_frame (%frame) if (%frame);
-                       %frame = ();
-                       $next_is_time_frame = 1;
-                       next;
-               }
-               if ($next_is_time_frame) {
-                       if (m/^\s*([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)/) {
-                               $frame{"Time"} = $2 + 0;
-                               if ($3 eq "host") {
-                                       $frame{"Direction"} = "Device";
-                               } else {
-                                       $frame{"Direction"} = "Host";
-                               }
-                       }
-                       $next_is_time_frame = 0;
-                       next;
-               }
-               if (m/^Frame\s+(\d+)/) {
-                       $frame{"Number"} = $1;
-                       next;
-               }
-               if (m/^USB\s+URB/) {
-                       next;
-               }
-               if (m/^\s+URB\s+id\:\s+(.*)/) {
-                       $frame{"ID"} = $1;
-                       next;
-               }
-               if (m/^\s+URB\s+type\:\s+([^\s]+)/) {
-                       $frame{"Type"} = $1;
-                       next;
-               }
-               if (m/^\s+URB\s+transfer\s+type\:\s+([^\s]+)/) {
-                       $frame{"TransferType"} = $1;
-                       next;
-               }
-               if (m/^\s+(Device|Endpoint|iConfiguration|idProduct|idVendor|iManufacturer|iSerialNumber|bcdDevice|bcdUSB|bDeviceClass|bDeviceProtocol|bDeviceSubClass|bMaxPacketSize0|bNumConfigurations|bNumInterfaces|bString|iProduct|wTotalLength)\:\s+(.*)/) {
-                       $frame{$1} = $2;
-                       next;
-               }
-
-
-               if (m/^\s+URB\s+bus\s+id\:\s+(.*)/) {
-                       $frame{"BusID"} = $1;
-                       next;
-               }
-               if (m/^\s+Device\s+setup\s+request\:\s+(.*)\s+\(/) {
-                       $frame{"SetupRequest"} = $1;
-                       next;
-               }
-               if (m/^\s+Data\:\s+(.*)\s+\(/) {
-                       $frame{"HasData"} = 1 if ($1 eq "present");
-                       next;
-               }
-               if (m/^\s+URB\s+status\:\s+([^\(]*)\s+\((.*)\)\s+\(/ || m/^\s+URB\s+status\:\s+([^\(]*)\s+\((.*)\)/) {
-                       $frame{"Status"} = $2;
-                       next;
-               }
-               if (m/^\s+URB\s+length\s+\[bytes\]\:\s+(.*)/) {
-                       $frame{"URBLength"} = $1;
-                       next;
-               }
-               if (m/^\s+Data\s+length\s+\[bytes\]\:\s+(.*)/) {
-                       $frame{"DataLength"} = $1;
-                       next;
-               }
-               if (m/^\s+wLANGID:.*(0x.*)/) {
-                       $frame{"wLANGID"} = $1;
-                       next;
-               }
-               if (m/^\s+bEndpointAddress\:\s+(.*)/) {
-                       # Probably need more parsing
-                       $frame{"EndpointAddress"} = $1;
-                       next;
-               }
-               if (m/^\s+\[(Request|Response)\s+in\:\s+(\d+)\]/) {
-                       $frame{"__RelatedTo"} = $2;
-                       $frame{"__RelationType"} = $1;
-                       next;
-               }
-               if (m/^\s+Configuration\s+bmAttributes\:\s+(.*)/) {
-                       $frame{"ConfigurationbmAttributes"} = $1;
-                       next;
-               }
-               if (m/^\s+bMaxPower\:\s+(.*)\s+\((.*)\)/) {
-                       $frame{"bMaxPower"} = $2;
-                       next;
-               }
-               next if (m/^\s+URB\s+setup/);
-               if (m/^\s+bmRequestType\:\s+(.*)/) {
-                       $frame{"bmRequestType"} = hex($1);
-                       next;
-               }
-               if (m/^\s+bmAttributes\:\s+(.*)/) {
-                       $frame{"bmAttributes"} = $1;
-                       next;
-               }
-               if (m/^\s+bRequest\:\s+(.*)\s+\((.*)\)/) {
-                       $frame{"bRequest"} = hex($2);
-                       next;
-               }
-               if (m/^\s+Descriptor\s+Index\:\s+(.*)/) {
-                       $frame{"DescriptorIndex"} = $1;
-                       next;
-               }
-               if (m/^\s+(bDescriptorType|bInterfaceClass)\:\s+(.*)\s+\((.*)\)/) {
-                       $frame{$1} = hex($3);
-                       next;
-               }
-               if (m/^\s+\[(bInterfaceClass)\:\s+(.*)\s+\((.*)\)\]/) {
-                       $frame{$1} = hex($3);
-                       next;
-               }
-               if (m/^\s+(bInterval|bInterfaceNumber|bInterfaceSubClass|bInterfaceProtocol)\:\s+(.*)/) {
-                       $frame{$1} = $2;
-                       next;
-               }
-               if (m/^\s+bAlternateSetting\:\s+(.*)/) {
-                       $frame{"bAlternateSetting"} = $1;
-                       next;
-               }
-               if (m/^\s+bConfigurationValue\:\s+(.*)/) {
-                       $frame{"bConfigurationValue"} = $1;
-                       next;
-               }
-               if (m/^\s+bLength\:\s+(.*)/) {
-                       $frame{"bLength"} = $1;
-                       next;
-               }
-               if (m/^\s+bNumEndpoints\:\s+(.*)/) {
-                       $frame{"bNumEndpoints"} = $1;
-                       next;
-               }
-               if (m/^\s+iInterface\:\s+(.*)/) {
-                       $frame{"iInterface"} = $1;
-                       next;
-               }
-               if (m/^\s+Language\s+Id\:\s+(.*)\s+\(/) {
-                       $frame{"LanguageId"} = $1;
-                       next;
-               }
-               if (m/^\s+wLength\:\s+(.*)/) {
-                       $frame{"wLength"} = $1;
-                       next;
-               }
-               if (m/^\s+wIndex\:\s+(.*)/) {
-                       $frame{"wIndex"} = $1;
-                       next;
-               }
-               if (m/^\s+wMaxPacketSize\:\s+(.*)/) {
-                       $frame{"wMaxPacketSize"} = $1;
-                       next;
-               }
-               if (m/^\s+wInterface\:\s+(.*)/) {
-                       $frame{"wInterface"} = $1;
-                       next;
-               }
-               if (m/^\s+Application\s+Data\:\s+(.*)/) {
-                       $frame{"ApplicationData"} = $1;
-                       next;
-               }
-               if (m/^\s+Frame\s+Number\:\s+(.*)/) {
-                       $frame{"FrameNumber"} = $1;
-                       next;
-               }
-               if (m/^\s+(Frame|Capture)\s+Length\:\s+(.*)\s+bytes/) {
-                       $frame{"$1Length"} = $2;
-                       next;
-               }
-
-               if (m/^\s+Arrival\s+Time:\s+(.*)/) {
-                       $frame{"ArrivalTime"} = str2time($1);
-                       next;
-               }
-               if (m/^\s+\[Time\s+from\s+request\:\s+(.*)\s+seconds\]/) {
-                       $frame{"TimeFromRequest"} = $1;
-                       next;
-               }
-               if (m/^\s+\[Time\s+delta\s+from\s+previous\s+(captured|displayed)\s+frame\:\s+(.*)\s+seconds\]/) {
-                       next;
-               }
-               if (m/^\s+\[Time\s+since\s+reference\s+or\s+first\s+frame\:\s+(.*)\s+seconds\]/) {
-                       next;
-               }
-               if (m/^\s+\[Frame\s+is\s+marked\:\s+(.*)/) {
-                       next;
-               }
-
-               # Remove some bitmap descriptions
-               next if (m/=\s+Direction\:/);
-               next if (m/=\s+Type\:/);
-               next if (m/=\s+Recipient\:/);
-               next if (m/=\s+Transfertype\:/);
-               next if (m/=\s+Synchronisationtype\:/);
-               next if (m/=\s+Behaviourtype\:/);
-               next if (m/=\s+Endpoint\s+Number\:/);
-               next if (m/=\s+Remote\s+Wakeup\:/);
-               next if (m/=\s+Self-Powered\:/);
-               next if (m/=\s+Must\s+be\s+1\:/);
-
-
-               # Prints unparsed strings
-               print "# Unparsed: $_" if ($debug);
-       }
-}
-
-wireshark_parser();
\ No newline at end of file