From c2c011b50f9a228a5388c228da37745f5d149d3f Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sat, 2 Oct 2010 22:58:31 -0300 Subject: [PATCH] contrib/pci_traffic: Add some patches for kvm to allow check PCI traffic Adds some patches that allow checking how PCI registers are accessed, by running the driver on a virtual kvm machine. Signed-off-by: Mauro Carvalho Chehab --- contrib/pci_traffic/README | 90 ++++++++++ contrib/pci_traffic/disconnect_usb.sh | 9 + .../kvm-allow-assigned-different-than-4K.patch | 185 +++++++++++++++++++++ contrib/pci_traffic/kvm-dev-assign-debug.patch | 64 +++++++ contrib/pci_traffic/write_log_to_file.patch | 55 ++++++ 5 files changed, 403 insertions(+) create mode 100644 contrib/pci_traffic/README create mode 100755 contrib/pci_traffic/disconnect_usb.sh create mode 100644 contrib/pci_traffic/kvm-allow-assigned-different-than-4K.patch create mode 100644 contrib/pci_traffic/kvm-dev-assign-debug.patch create mode 100644 contrib/pci_traffic/write_log_to_file.patch diff --git a/contrib/pci_traffic/README b/contrib/pci_traffic/README new file mode 100644 index 0000000..2607726 --- /dev/null +++ b/contrib/pci_traffic/README @@ -0,0 +1,90 @@ +INTRODUCTION +============ +Sometimes, it is useful to check what the driver is doing with +the PCI registers. + +As it took me some time to get the right info from the Intenet, +and to write a few patches to enable debug, and to write them +to a file, I'm documenting here the procedure I used, as it +may be useful for the others. + +One of the ways to see what a driver is doing with PCI registers +is to write several debug printk's. Another way is to enable PCI +device access at a VM machine, and catch the PCI emulation logs. + +To use the second case, one of the ways is to modify the qemu-kvm +package. In my case, I wrote some patches for +qemu-kvm-0.12.1.2-2.113.el6.x86_64 (shipped with RHEL6), but they +probably apply on other versions of qemu. + +Be careful: in order to get the PCI writes, the patches I'm using +intercepts all calls to PCI write/read to the mapped devices. This +is slow, and may cause machine hangs. If you're willing to use it, +you'll likely need to do on a separate machine, used just for +tests. On my test machine, they work fine for digital TV, but hangs +the machine with analog TV (probably because digital streams use +compressed traffic). + +PATCHES FOR QEMU-KVM +==================== + +There are a few patches that are needed to allow getting a kvm log +from the PCI traffic. + +The patches should be applied on this order: + +kvm-allow-assigned-different-than-4K.patch +kvm-dev-assign-debug.patch +write_log_to_file.patch + +1) kvm-allow-assigned-different-than-4K.patch + +This patch came from qemu-kvm git tree. It will likely be available +on some stable version soon. It is needed for some devices. + +2) kvm-dev-assign-debug.patch +This patch were written by me. It basically enables some debug +logs. It is licensed with GPLv2; + +3) write_log_to_file.patch + +This patch were written by me. It changes the debug logic to write +logs on a file, instead of stderr. The file is: /var/log/qemu_hw_pci.log. +You'll need to create it previously, change its ownership to match +qemu user, and fix selinux permissions (if you're using selinux). +In the case of RHEL6, I'm using: + +-rw-r--r--. qemu qemu system_u:system_r:svirt_t:s0 qemu_hw_pci.log + +IRQ CONFLICTS +============= + +You need to be sure that IRQ's are not shared between the device you +want to sniff and any other device, otherwise, kvm won't allow you +to see the physical device. This is (currently) a kvm restriction. + +You may check it by looking at /proc/interrupts. + +In my case, ehci_hcd and uhci_hcd shares interrrupts with devices I +plug on my PCI slots. So, I wrote a script to disconnect all USB buses +from my test machine. The script is: + + disconnect_usb.sh + +MAPPING THE PHYSICAL DEVICE ON QEMU +=================================== + +You'll need to use an option for KVM to map the PCI device at the +machine. In my case, I'm using those lines: + +-device pci-assign,host=37:09.0,id=hostdev0,configfd=22,bus=pci.0,addr=0x6 +-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 + +The actual line depends on the PCI device that you're using. The above +example assumes a device at PCI address 37:09.0. + +You may also use virt-manager to configure physical access to your PCI +device. + +I hope this may be useful for you. +Mauro Carvalho Chehab diff --git a/contrib/pci_traffic/disconnect_usb.sh b/contrib/pci_traffic/disconnect_usb.sh new file mode 100755 index 0000000..f37a30f --- /dev/null +++ b/contrib/pci_traffic/disconnect_usb.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +for i in 0 1 2 3 4 5 6 7; do + echo "0000:00:1d.$i" >/sys/bus/pci/drivers/ehci_hcd/unbind + echo "0000:00:1d.$i" >/sys/bus/pci/drivers/uhci_hcd/unbind + echo "0000:00:1a.$i" >/sys/bus/pci/drivers/ehci_hcd/unbind + echo "0000:00:1a.$i" >/sys/bus/pci/drivers/uhci_hcd/unbind + echo "0000:00:1d.$i" >/sys/bus/pci/drivers/pci-stub/bind +done diff --git a/contrib/pci_traffic/kvm-allow-assigned-different-than-4K.patch b/contrib/pci_traffic/kvm-allow-assigned-different-than-4K.patch new file mode 100644 index 0000000..bcb1ad2 --- /dev/null +++ b/contrib/pci_traffic/kvm-allow-assigned-different-than-4K.patch @@ -0,0 +1,185 @@ +commit 8845b72377a42443b249360b9cd3d6f1e8cfb097 +Author: Alexander Graf +Date: Thu Dec 17 16:04:44 2009 +0100 + + Enable non page boundary BAR device assignment + + While trying to get device passthrough working with an emulex hba, kvm + refused to pass it through because it has a BAR of 256 bytes: + + Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K] + Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256] + Region 4: I/O ports at b100 [size=256] + + Since the page boundary is an arbitrary optimization to allow 1:1 mapping of + physical to virtual addresses, we can still take the old MMIO callback route. + + So let's add a second code path that allows for size & 0xFFF != 0 sized regions + by looping it through userspace. + + I verified that it works by passing through an e1000 with this additional patch + applied and the card acted the same way it did without this patch: + + map_func = assigned_dev_iomem_map; + - if (cur_region->size & 0xFFF) { + + if (i != PCI_ROM_SLOT){ + fprintf(stderr, "PCI region %d at address 0x%llx " + + Signed-off-by: Alexander Graf + Signed-off-by: Avi Kivity + +diff --git a/hw/device-assignment.c b/hw/device-assignment.c +index 5422e9a..c99d986 100644 +--- a/hw/device-assignment.c ++++ b/hw/device-assignment.c +@@ -158,6 +158,105 @@ static uint32_t assigned_dev_ioport_readl(void *opaque, uint32_t addr) + return assigned_dev_ioport_rw(opaque, addr, 4, NULL); + } + ++static uint32_t slow_bar_readb(void *opaque, target_phys_addr_t addr) ++{ ++ AssignedDevRegion *d = opaque; ++ uint8_t *in = d->u.r_virtbase + addr; ++ uint32_t r; ++ ++ r = *in; ++ DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r); ++ ++ return r; ++} ++ ++static uint32_t slow_bar_readw(void *opaque, target_phys_addr_t addr) ++{ ++ AssignedDevRegion *d = opaque; ++ uint16_t *in = d->u.r_virtbase + addr; ++ uint32_t r; ++ ++ r = *in; ++ DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r); ++ ++ return r; ++} ++ ++static uint32_t slow_bar_readl(void *opaque, target_phys_addr_t addr) ++{ ++ AssignedDevRegion *d = opaque; ++ uint32_t *in = d->u.r_virtbase + addr; ++ uint32_t r; ++ ++ r = *in; ++ DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r); ++ ++ return r; ++} ++ ++static void slow_bar_writeb(void *opaque, target_phys_addr_t addr, uint32_t val) ++{ ++ AssignedDevRegion *d = opaque; ++ uint8_t *out = d->u.r_virtbase + addr; ++ ++ DEBUG("slow_bar_writeb addr=0x" TARGET_FMT_plx " val=0x%02x\n", addr, val); ++ *out = val; ++} ++ ++static void slow_bar_writew(void *opaque, target_phys_addr_t addr, uint32_t val) ++{ ++ AssignedDevRegion *d = opaque; ++ uint16_t *out = d->u.r_virtbase + addr; ++ ++ DEBUG("slow_bar_writew addr=0x" TARGET_FMT_plx " val=0x%04x\n", addr, val); ++ *out = val; ++} ++ ++static void slow_bar_writel(void *opaque, target_phys_addr_t addr, uint32_t val) ++{ ++ AssignedDevRegion *d = opaque; ++ uint32_t *out = d->u.r_virtbase + addr; ++ ++ DEBUG("slow_bar_writel addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, val); ++ *out = val; ++} ++ ++static CPUWriteMemoryFunc * const slow_bar_write[] = { ++ &slow_bar_writeb, ++ &slow_bar_writew, ++ &slow_bar_writel ++}; ++ ++static CPUReadMemoryFunc * const slow_bar_read[] = { ++ &slow_bar_readb, ++ &slow_bar_readw, ++ &slow_bar_readl ++}; ++ ++static void assigned_dev_iomem_map_slow(PCIDevice *pci_dev, int region_num, ++ pcibus_t e_phys, pcibus_t e_size, ++ int type) ++{ ++ AssignedDevice *r_dev = container_of(pci_dev, AssignedDevice, dev); ++ AssignedDevRegion *region = &r_dev->v_addrs[region_num]; ++ PCIRegion *real_region = &r_dev->real_device.regions[region_num]; ++ int m; ++ ++ DEBUG("slow map type %i\n", type); ++ m = cpu_register_io_memory(slow_bar_read, slow_bar_write, region); ++ cpu_register_physical_memory(e_phys, e_size, m); ++ ++ /* MSI-X MMIO page */ ++ if ((e_size > 0) && ++ real_region->base_addr <= r_dev->msix_table_addr && ++ real_region->base_addr + real_region->size >= r_dev->msix_table_addr) { ++ int offset = r_dev->msix_table_addr - real_region->base_addr; ++ ++ cpu_register_physical_memory(e_phys + offset, ++ TARGET_PAGE_SIZE, r_dev->mmio_index); ++ } ++} ++ + static void assigned_dev_iomem_map(PCIDevice *pci_dev, int region_num, + pcibus_t e_phys, pcibus_t e_size, int type) + { +@@ -456,15 +555,22 @@ static int assigned_dev_register_regions(PCIRegion *io_regions, + + /* handle memory io regions */ + if (cur_region->type & IORESOURCE_MEM) { ++ int slow_map = 0; + int t = cur_region->type & IORESOURCE_PREFETCH + ? PCI_BASE_ADDRESS_MEM_PREFETCH + : PCI_BASE_ADDRESS_SPACE_MEMORY; ++ + if (cur_region->size & 0xFFF) { +- fprintf(stderr, "Unable to assign device: PCI region %d " +- "at address 0x%llx has size 0x%x, " +- " which is not a multiple of 4K\n", ++ fprintf(stderr, "PCI region %d at address 0x%llx " ++ "has size 0x%x, which is not a multiple of 4K. " ++ "You might experience some performance hit due to that.\n", + i, (unsigned long long)cur_region->base_addr, + cur_region->size); ++ slow_map = 1; ++ } ++ ++ if (slow_map && (i == PCI_ROM_SLOT)) { ++ fprintf(stderr, "ROM not aligned - can't continue\n"); + return -1; + } + +@@ -480,7 +586,7 @@ static int assigned_dev_register_regions(PCIRegion *io_regions, + } else { + pci_dev->v_addrs[i].u.r_virtbase = + mmap(NULL, +- (cur_region->size + 0xFFF) & 0xFFFFF000, ++ cur_region->size, + PROT_WRITE | PROT_READ, MAP_SHARED, + cur_region->resource_fd, (off_t) 0); + } +@@ -507,7 +613,8 @@ static int assigned_dev_register_regions(PCIRegion *io_regions, + + pci_register_bar((PCIDevice *) pci_dev, i, + cur_region->size, t, +- assigned_dev_iomem_map); ++ slow_map ? assigned_dev_iomem_map_slow ++ : assigned_dev_iomem_map); + continue; + } else { + /* handle port io regions */ diff --git a/contrib/pci_traffic/kvm-dev-assign-debug.patch b/contrib/pci_traffic/kvm-dev-assign-debug.patch new file mode 100644 index 0000000..1f776c5 --- /dev/null +++ b/contrib/pci_traffic/kvm-dev-assign-debug.patch @@ -0,0 +1,64 @@ +diff --git a/hw/device-assignment.c b/hw/device-assignment.c +index c99d986..4af9ff6 100644 +--- a/hw/device-assignment.c ++++ b/hw/device-assignment.c +@@ -47,7 +47,7 @@ + #define IORESOURCE_DMA 0x00000800 + #define IORESOURCE_PREFETCH 0x00001000 /* No side effects */ + +-/* #define DEVICE_ASSIGNMENT_DEBUG 1 */ ++#define DEVICE_ASSIGNMENT_DEBUG 1 + + #ifdef DEVICE_ASSIGNMENT_DEBUG + #define DEBUG(fmt, ...) \ +@@ -267,10 +267,10 @@ static void assigned_dev_iomem_map(PCIDevice *pci_dev, int region_num, + pcibus_t old_esize = region->e_size; + int first_map = (region->e_size == 0); + int ret = 0; +- ++#if 0 + DEBUG("e_phys=%08x r_virt=%p type=%d len=%08x region_num=%d \n", + e_phys, region->u.r_virtbase, type, e_size, region_num); +- ++#endif + region->e_physbase = e_phys; + region->e_size = e_size; + +@@ -344,10 +344,10 @@ static void assigned_dev_ioport_map(PCIDevice *pci_dev, int region_num, + + region->e_physbase = addr; + region->e_size = size; +- ++#if 0 + DEBUG("e_phys=0x%x r_baseport=%x type=0x%x len=%d region_num=%d \n", + addr, region->u.r_baseport, type, size, region_num); +- ++#endif + if (first_map && region->region->resource_fd < 0) { + struct ioperm_data *data; + +@@ -561,11 +561,13 @@ static int assigned_dev_register_regions(PCIRegion *io_regions, + : PCI_BASE_ADDRESS_SPACE_MEMORY; + + if (cur_region->size & 0xFFF) { ++#if 0 + fprintf(stderr, "PCI region %d at address 0x%llx " + "has size 0x%x, which is not a multiple of 4K. " + "You might experience some performance hit due to that.\n", + i, (unsigned long long)cur_region->base_addr, + cur_region->size); ++#endif + slow_map = 1; + } + +@@ -1341,8 +1343,8 @@ static void msix_mmio_writel(void *opaque, + unsigned int offset = addr & 0xfff; + void *page = adev->msix_table_page; + +- DEBUG("write to MSI-X entry table mmio offset 0x%lx, val 0x%lx\n", +- addr, val); ++ DEBUG("write to MSI-X entry table mmio offset 0x%lx, val 0x%ux\n", ++ (unsigned long)addr, val); + memcpy((void *)((char *)page + offset), &val, 4); + } + diff --git a/contrib/pci_traffic/write_log_to_file.patch b/contrib/pci_traffic/write_log_to_file.patch new file mode 100644 index 0000000..1d18613 --- /dev/null +++ b/contrib/pci_traffic/write_log_to_file.patch @@ -0,0 +1,55 @@ +diff --git a/hw/device-assignment.c b/hw/device-assignment.c +index 4af9ff6..aae2d8a 100644 +--- a/hw/device-assignment.c ++++ b/hw/device-assignment.c +@@ -25,8 +25,10 @@ + * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) + * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) + */ ++ + #include + #include ++#include + #include + #include + #include +@@ -50,9 +52,15 @@ + #define DEVICE_ASSIGNMENT_DEBUG 1 + + #ifdef DEVICE_ASSIGNMENT_DEBUG +-#define DEBUG(fmt, ...) \ +- do { \ +- fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__); \ ++static FILE *debugfp = NULL; ++ ++#define DEBUG(fmt, ...) \ ++ do { \ ++ if (!debugfp) \ ++ debugfp = fopen("/var/log/qemu_hw_pci.log", "a+"); \ ++ if (debugfp) \ ++ fprintf(debugfp, "%ld %s: " fmt, (long)time(NULL), \ ++ __func__ , __VA_ARGS__); \ + } while (0) + #else + #define DEBUG(fmt, ...) do { } while(0) +@@ -560,6 +568,10 @@ static int assigned_dev_register_regions(PCIRegion *io_regions, + ? PCI_BASE_ADDRESS_MEM_PREFETCH + : PCI_BASE_ADDRESS_SPACE_MEMORY; + ++#ifdef DEVICE_ASSIGNMENT_DEBUG ++ slow_map = 1; ++#endif ++ + if (cur_region->size & 0xFFF) { + #if 0 + fprintf(stderr, "PCI region %d at address 0x%llx " +@@ -1453,6 +1465,9 @@ static int assigned_initfn(struct PCIDevice *pci_dev) + if (assigned_dev_register_msix_mmio(dev)) + goto assigned_out; + ++ DEBUG("Assigned device 0000:%02x:%02x.%01x/rom", ++ dev->host.bus, dev->host.dev, dev->host.func); ++ + assigned_dev_load_option_rom(dev); + QLIST_INSERT_HEAD(&devs, dev, next); + return 0; -- 2.7.4