# deadlock_detector Detects potential deadlocks (lock order inversions)
# on a running process. For Linux, uses BCC, eBPF.
#
-# USAGE: deadlock_detector.py [-h] [--dump-graph DUMP_GRAPH]
-# [--lock-symbols LOCK_SYMBOLS]
+# USAGE: deadlock_detector.py [-h] [--binary BINARY] [--dump-graph DUMP_GRAPH]
+# [--verbose] [--lock-symbols LOCK_SYMBOLS]
# [--unlock-symbols UNLOCK_SYMBOLS]
-# binary pid
+# pid
#
# This traces pthread mutex lock and unlock calls to build a directed graph
# representing the mutex wait graph:
# the cycle of mutexes and the stack traces where each mutex was acquired, and
# then exit.
#
+# This program can only find potential deadlocks that occur while the program
+# is tracing the process. It cannot find deadlocks that may have occurred
+# before the program was attached to the process.
+#
+# Since this traces all mutex lock and unlock events and all thread creation
+# events on the traced process, the overhead of this bpf program can be very
+# high if the process has many threads and mutexes. You should only run this on
+# a process where the slowdown is acceptable.
+#
# Note: This tool does not work for shared mutexes or recursive mutexes.
#
# For shared (read-write) mutexes, a deadlock requires a cycle in the wait
absolute_import, division, unicode_literals, print_function
)
from bcc import BPF
+from collections import defaultdict
import argparse
import json
-import networkx
-from networkx.readwrite import json_graph
+import os
+import subprocess
import sys
import time
+class DiGraph(object):
+ '''
+ Adapted from networkx: http://networkx.github.io/
+ Represents a directed graph. Edges can store (key, value) attributes.
+ '''
+
+ def __init__(self):
+ # Map of node -> set of nodes
+ self.adjacency_map = {}
+ # Map of (node1, node2) -> map string -> arbitrary attribute
+ # This will not be copied in subgraph()
+ self.attributes_map = {}
+
+ def neighbors(self, node):
+ return self.adjacency_map.get(node, set())
+
+ def edges(self):
+ edges = []
+ for node, neighbors in self.adjacency_map.items():
+ for neighbor in neighbors:
+ edges.append((node, neighbor))
+ return edges
+
+ def nodes(self):
+ return self.adjacency_map.keys()
+
+ def attributes(self, node1, node2):
+ return self.attributes_map[(node1, node2)]
+
+ def add_edge(self, node1, node2, **kwargs):
+ if node1 not in self.adjacency_map:
+ self.adjacency_map[node1] = set()
+ if node2 not in self.adjacency_map:
+ self.adjacency_map[node2] = set()
+ self.adjacency_map[node1].add(node2)
+ self.attributes_map[(node1, node2)] = kwargs
+
+ def remove_node(self, node):
+ self.adjacency_map.pop(node, None)
+ for _, neighbors in self.adjacency_map.items():
+ neighbors.discard(node)
+
+ def subgraph(self, nodes):
+ graph = DiGraph()
+ for node in nodes:
+ for neighbor in self.neighbors(node):
+ if neighbor in nodes:
+ graph.add_edge(node, neighbor)
+ return graph
+
+ def node_link_data(self):
+ '''
+ Returns the graph as a dictionary in a format that can be
+ serialized.
+ '''
+ data = {
+ 'directed': True,
+ 'multigraph': False,
+ 'graph': {},
+ 'links': [],
+ 'nodes': [],
+ }
+
+ # Do one pass to build a map of node -> position in nodes
+ node_to_number = {}
+ for node in self.adjacency_map.keys():
+ node_to_number[node] = len(data['nodes'])
+ data['nodes'].append({'id': node})
+
+ # Do another pass to build the link information
+ for node, neighbors in self.adjacency_map.items():
+ for neighbor in neighbors:
+ link = self.attributes_map[(node, neighbor)].copy()
+ link['source'] = node_to_number[node]
+ link['target'] = node_to_number[neighbor]
+ data['links'].append(link)
+ return data
+
+
+def strongly_connected_components(G):
+ '''
+ Adapted from networkx: http://networkx.github.io/
+ Parameters
+ ----------
+ G : DiGraph
+ Returns
+ -------
+ comp : generator of sets
+ A generator of sets of nodes, one for each strongly connected
+ component of G.
+ '''
+ preorder = {}
+ lowlink = {}
+ scc_found = {}
+ scc_queue = []
+ i = 0 # Preorder counter
+ for source in G.nodes():
+ if source not in scc_found:
+ queue = [source]
+ while queue:
+ v = queue[-1]
+ if v not in preorder:
+ i = i + 1
+ preorder[v] = i
+ done = 1
+ v_nbrs = G.neighbors(v)
+ for w in v_nbrs:
+ if w not in preorder:
+ queue.append(w)
+ done = 0
+ break
+ if done == 1:
+ lowlink[v] = preorder[v]
+ for w in v_nbrs:
+ if w not in scc_found:
+ if preorder[w] > preorder[v]:
+ lowlink[v] = min([lowlink[v], lowlink[w]])
+ else:
+ lowlink[v] = min([lowlink[v], preorder[w]])
+ queue.pop()
+ if lowlink[v] == preorder[v]:
+ scc_found[v] = True
+ scc = {v}
+ while (
+ scc_queue and preorder[scc_queue[-1]] > preorder[v]
+ ):
+ k = scc_queue.pop()
+ scc_found[k] = True
+ scc.add(k)
+ yield scc
+ else:
+ scc_queue.append(v)
+
+
+def simple_cycles(G):
+ '''
+ Adapted from networkx: http://networkx.github.io/
+ Parameters
+ ----------
+ G : DiGraph
+ Returns
+ -------
+ cycle_generator: generator
+ A generator that produces elementary cycles of the graph.
+ Each cycle is represented by a list of nodes along the cycle.
+ '''
+
+ def _unblock(thisnode, blocked, B):
+ stack = set([thisnode])
+ while stack:
+ node = stack.pop()
+ if node in blocked:
+ blocked.remove(node)
+ stack.update(B[node])
+ B[node].clear()
+
+ # Johnson's algorithm requires some ordering of the nodes.
+ # We assign the arbitrary ordering given by the strongly connected comps
+ # There is no need to track the ordering as each node removed as processed.
+ # save the actual graph so we can mutate it here
+ # We only take the edges because we do not want to
+ # copy edge and node attributes here.
+ subG = G.subgraph(G.nodes())
+ sccs = list(strongly_connected_components(subG))
+ while sccs:
+ scc = sccs.pop()
+ # order of scc determines ordering of nodes
+ startnode = scc.pop()
+ # Processing node runs 'circuit' routine from recursive version
+ path = [startnode]
+ blocked = set() # vertex: blocked from search?
+ closed = set() # nodes involved in a cycle
+ blocked.add(startnode)
+ B = defaultdict(set) # graph portions that yield no elementary circuit
+ stack = [(startnode, list(subG.neighbors(startnode)))]
+ while stack:
+ thisnode, nbrs = stack[-1]
+ if nbrs:
+ nextnode = nbrs.pop()
+ if nextnode == startnode:
+ yield path[:]
+ closed.update(path)
+ elif nextnode not in blocked:
+ path.append(nextnode)
+ stack.append((nextnode, list(subG.neighbors(nextnode))))
+ closed.discard(nextnode)
+ blocked.add(nextnode)
+ continue
+ # done with nextnode... look for more neighbors
+ if not nbrs: # no more nbrs
+ if thisnode in closed:
+ _unblock(thisnode, blocked, B)
+ else:
+ for nbr in subG.neighbors(thisnode):
+ if thisnode not in B[nbr]:
+ B[nbr].add(thisnode)
+ stack.pop()
+ path.pop()
+ # done processing this node
+ subG.remove_node(startnode)
+ H = subG.subgraph(scc) # make smaller to avoid work in SCC routine
+ sccs.extend(list(strongly_connected_components(H)))
+
+
def find_cycle(graph):
'''
Looks for a cycle in the graph. If found, returns the first cycle.
If nodes a1, a2, ..., an are in a cycle, then this returns:
-
- [(a1,a2), (a2,a3), ... (an-1,an), (an, a1)]
-
+ [(a1,a2), (a2,a3), ... (an-1,an), (an, a1)]
Otherwise returns an empty list.
'''
- cycles = list(networkx.simple_cycles(graph))
+ cycles = list(simple_cycles(graph))
if cycles:
nodes = cycles[0]
nodes.append(nodes[0])
return []
-def find_cycle_and_print(graph, thread_to_parent, print_stack_trace_fn):
+def print_cycle(binary, graph, edges, thread_info, print_stack_trace_fn):
'''
- Detects if there is a cycle in the mutex graph. If there is, dump
- the following information and return True. Otherwise returns False.
+ Prints the cycle in the mutex graph in the following format:
Potential Deadlock Detected!
Thread T was created here:
[ stack trace ]
'''
- edges = find_cycle(graph)
- if not edges:
- return False
# List of mutexes in the cycle, first and last repeated
nodes_in_order = []
node_addr_to_name = {}
for counter, (m, n) in enumerate(edges):
nodes_in_order.append(m)
- node_addr_to_name[m] = 'Mutex M%d (0x%016x)' % (counter, m)
+ # For global or static variables, try to symbolize the mutex address.
+ symbol = symbolize_with_objdump(binary, m)
+ if symbol:
+ symbol += ' '
+ node_addr_to_name[m] = 'Mutex M%d (%s0x%016x)' % (counter, symbol, m)
nodes_in_order.append(nodes_in_order[0])
print('----------------\nPotential Deadlock Detected!\n')
# For each edge in the cycle, print where the two mutexes were held
for (m, n) in edges:
- thread_pid = graph[m][n]['thread_pid']
- thread_comm = graph[m][n]['thread_comm']
- first_mutex_stack_id = graph[m][n]['first_mutex_stack_id']
- second_mutex_stack_id = graph[m][n]['second_mutex_stack_id']
+ thread_pid = graph.attributes(m, n)['thread_pid']
+ thread_comm = graph.attributes(m, n)['thread_comm']
+ first_mutex_stack_id = graph.attributes(m, n)['first_mutex_stack_id']
+ second_mutex_stack_id = graph.attributes(m, n)['second_mutex_stack_id']
thread_pids.add(thread_pid)
print(
'%s acquired here while holding %s in Thread %d (%s):' % (
# Print where the threads were created, if available
for thread_pid in thread_pids:
- parent_pid, stack_id, parent_comm = thread_to_parent.get(
+ parent_pid, stack_id, parent_comm = thread_info.get(
thread_pid, (None, None, None)
)
if parent_pid:
thread_pid
)
print('')
- return True
-def strlist(s):
+def symbolize_with_objdump(binary, addr):
'''
- Given a comma-separated string, returns a list of substrings
+ Searches the biniary for the address using objdump. Returns the symbol if
+ it is found, otherwise returns empty string.
'''
+ try:
+ command = (
+ 'objdump -tT %s | grep %x | awk {\'print $NF\'} | c++filt' %
+ (binary, addr)
+ )
+ output = subprocess.check_output(command, shell=True)
+ return output.decode('utf-8').strip()
+ except subprocess.CalledProcessError:
+ return ''
+
+
+def strlist(s):
+ '''Given a comma-separated string, returns a list of substrings'''
return s.strip().split(',')
' Must be run as root.'
)
)
- parser.add_argument('binary', type=str, help='Absolute path to binary')
parser.add_argument('pid', type=int, help='Pid to trace')
+ # Binaries with `:` in the path will fail to attach uprobes on kernels
+ # running without this patch: https://lkml.org/lkml/2017/1/13/585.
+ # Symlinks to the binary without `:` in the path can get around this issue.
+ parser.add_argument(
+ '--binary',
+ type=str,
+ default='',
+ help='If set, use this as the path to the binary for the process.',
+ )
parser.add_argument(
'--dump-graph',
type=str,
help='If set, this will dump the mutex graph to the specified file.',
)
parser.add_argument(
+ '--verbose',
+ action='store_true',
+ help='Print statistics about the mutex wait graph.',
+ )
+ parser.add_argument(
'--lock-symbols',
type=strlist,
default=['pthread_mutex_lock'],
'pthread_mutex_unlock',
)
args = parser.parse_args()
+ if not args.binary:
+ try:
+ args.binary = os.readlink('/proc/%d/exe' % args.pid)
+ except OSError as e:
+ print('%s. Is the process (pid=%d) running?' % (str(e), args.pid))
+ sys.exit(1)
+
bpf = BPF(src_file='deadlock_detector.c')
# Trace where threads are created
bpf.attach_kretprobe(
- event='sys_clone',
- fn_name='trace_clone',
- pid=args.pid
+ event='sys_clone', fn_name='trace_clone', pid=args.pid
)
# We must trace unlock first, otherwise in the time we attached the probe
- # on lock() and have not yet attached the probe on unlock(), a thread
- # can acquire multiple mutexes and released them, but the release
- # events will not be traced, resulting in noisy reports.
+ # on lock() and have not yet attached the probe on unlock(), a thread can
+ # acquire mutexes and release them, but the release events will not be
+ # traced, resulting in noisy reports.
for symbol in args.unlock_symbols:
try:
bpf.attach_uprobe(
pid=args.pid,
)
except Exception as e:
- print(e)
+ print('%s. Failed to attach to symbol: %s' % (str(e), symbol))
sys.exit(1)
for symbol in args.lock_symbols:
try:
pid=args.pid,
)
except Exception as e:
- print(e)
+ print('%s. Failed to attach to symbol: %s' % (str(e), symbol))
sys.exit(1)
def print_stack_trace(stack_id):
- '''
- Closure that prints the symbolized stack trace. Captures: `bpf`, `args`
- '''
+ '''Closure that prints the symbolized stack trace.'''
for addr in bpf.get_table('stack_traces').walk(stack_id):
line = bpf.sym(addr, args.pid)
+ # Try to symbolize with objdump if we cannot with bpf.
+ if line == '[unknown]':
+ symbol = symbolize_with_objdump(args.binary, addr)
+ if symbol:
+ line = symbol
print('@ %016x %s' % (addr, line))
print('Tracing... Hit Ctrl-C to end.')
while True:
try:
+ # Map of child thread pid -> parent info
+ thread_info = {
+ child.value: (parent.parent_pid, parent.stack_id, parent.comm)
+ for child, parent in bpf.get_table('thread_to_parent').items()
+ }
+
# Mutex wait directed graph. Nodes are mutexes. Edge (A,B) exists
# if there exists some thread T where lock(A) was called and
# lock(B) was called before unlock(A) was called.
- graph = networkx.DiGraph()
+ graph = DiGraph()
for key, leaf in bpf.get_table('edges').items():
graph.add_edge(
key.mutex1,
first_mutex_stack_id=leaf.mutex1_stack_id,
second_mutex_stack_id=leaf.mutex2_stack_id,
)
+ if args.verbose:
+ print(
+ 'Mutexes: %d, Edges: %d' %
+ (len(graph.nodes()), len(graph.edges()))
+ )
if args.dump_graph:
with open(args.dump_graph, 'w') as f:
- data = json_graph.node_link_data(graph)
+ data = graph.node_link_data()
f.write(json.dumps(data, indent=2))
- # Map of child thread pid -> parent info
- thread_to_parent_dict = {
- child.value: (parent.parent_pid, parent.stack_id, parent.comm)
- for child, parent in bpf.get_table('thread_to_parent').items()
- }
-
- start = time.time()
- has_cycle = find_cycle_and_print(
- graph, thread_to_parent_dict, print_stack_trace
- )
- print(
- 'Nodes: %d, Edges: %d, Looking for cycle took %f seconds' %
- (len(graph.nodes()), len(graph.edges()), (time.time() - start))
- )
- if has_cycle:
+ cycle = find_cycle(graph)
+ if cycle:
+ print_cycle(
+ args.binary, graph, cycle, thread_info, print_stack_trace
+ )
sys.exit(1)
time.sleep(1)
program will dump the cycle of mutexes, dump the stack traces where each mutex
was acquired, and then exit.
-This program can only find potential deadlocks that occur while the program is
-tracing the process. It cannot find deadlocks that may have occurred before the
-program was attached to the process.
+This program can only find potential deadlocks that occur while the program
+is tracing the process. It cannot find deadlocks that may have occurred
+before the program was attached to the process.
+
+Since this traces all mutex lock and unlock events and all thread creation
+events on the traced process, the overhead of this bpf program can be very
+high if the process has many threads and mutexes. You should only run this on
+a process where the slowdown is acceptable.
Note: This tool does not work for shared mutexes or recursive mutexes.
potential deadlocks that involve only one mutex.
-# ./deadlock_detector.py /path/to/program/with/lockinversion $(pidof lockinversion)
+# ./deadlock_detector.py $(pidof lockinversion)
Tracing... Hit Ctrl-C to end.
-Nodes: 0, Edges: 0, Looking for cycle took 0.000056 seconds
-Nodes: 0, Edges: 0, Looking for cycle took 0.000062 seconds
-Nodes: 0, Edges: 0, Looking for cycle took 0.000070 seconds
-Nodes: 0, Edges: 0, Looking for cycle took 0.000071 seconds
-Nodes: 0, Edges: 0, Looking for cycle took 0.000066 seconds
-Nodes: 0, Edges: 0, Looking for cycle took 0.000066 seconds
----------------
Potential Deadlock Detected!
-Cycle in lock order graph: Mutex M0 (0x00007ffccd7ab140) => Mutex M1 (0x00007ffccd7ab0b0) => Mutex M2 (0x00007ffccd7ab0e0) => Mutex M3 (0x00007ffccd7ab110) => Mutex M0 (0x00007ffccd7ab140)
-
-Mutex M1 (0x00007ffccd7ab0b0) acquired here while holding Mutex M0 (0x00007ffccd7ab140) in Thread 3120373 (lockinversion):
-@ 00000000004024d0 [unknown]
-@ 0000000000406f4e std::mutex::lock()
-@ 0000000000407250 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
-@ 0000000000402ecc main::{lambda()#4}::operator()() const
-@ 0000000000406cc4 void std::_Bind_simple<main::{lambda()#4} ()>::_M_invoke<>(std::_Index_tuple<>)
-@ 0000000000406aab std::_Bind_simple<main::{lambda()#4} ()>::operator()()
-@ 000000000040689a std::thread::_Impl<std::_Bind_simple<main::{lambda()#4} ()> >::_M_run()
-@ 00007f9f9791f4e1 execute_native_thread_routine
-@ 00007f9f9809e7f1 start_thread
-@ 00007f9f9736046d __clone
-
-Mutex M0 (0x00007ffccd7ab140) previously acquired by the same Thread 3120373 (lockinversion) here:
-@ 00000000004024d0 [unknown]
-@ 0000000000406f4e std::mutex::lock()
-@ 0000000000407250 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
-@ 0000000000402eb6 main::{lambda()#4}::operator()() const
-@ 0000000000406cc4 void std::_Bind_simple<main::{lambda()#4} ()>::_M_invoke<>(std::_Index_tuple<>)
-@ 0000000000406aab std::_Bind_simple<main::{lambda()#4} ()>::operator()()
-@ 000000000040689a std::thread::_Impl<std::_Bind_simple<main::{lambda()#4} ()> >::_M_run()
-@ 00007f9f9791f4e1 execute_native_thread_routine
-@ 00007f9f9809e7f1 start_thread
-@ 00007f9f9736046d __clone
-
-Mutex M2 (0x00007ffccd7ab0e0) acquired here while holding Mutex M1 (0x00007ffccd7ab0b0) in Thread 3120370 (lockinversion):
-@ 00000000004024d0 [unknown]
-@ 0000000000406f4e std::mutex::lock()
-@ 0000000000407250 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
-@ 0000000000402d6a main::{lambda()#1}::operator()() const
-@ 0000000000406dea void std::_Bind_simple<main::{lambda()#1} ()>::_M_invoke<>(std::_Index_tuple<>)
-@ 0000000000406b17 std::_Bind_simple<main::{lambda()#1} ()>::operator()()
-@ 00000000004068f4 std::thread::_Impl<std::_Bind_simple<main::{lambda()#1} ()> >::_M_run()
-@ 00007f9f9791f4e1 execute_native_thread_routine
-@ 00007f9f9809e7f1 start_thread
-@ 00007f9f9736046d __clone
-
-Mutex M1 (0x00007ffccd7ab0b0) previously acquired by the same Thread 3120370 (lockinversion) here:
-@ 00000000004024d0 [unknown]
-@ 0000000000406f4e std::mutex::lock()
-@ 0000000000407250 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
-@ 0000000000402d53 main::{lambda()#1}::operator()() const
-@ 0000000000406dea void std::_Bind_simple<main::{lambda()#1} ()>::_M_invoke<>(std::_Index_tuple<>)
-@ 0000000000406b17 std::_Bind_simple<main::{lambda()#1} ()>::operator()()
-@ 00000000004068f4 std::thread::_Impl<std::_Bind_simple<main::{lambda()#1} ()> >::_M_run()
-@ 00007f9f9791f4e1 execute_native_thread_routine
-@ 00007f9f9809e7f1 start_thread
-@ 00007f9f9736046d __clone
-
-Mutex M3 (0x00007ffccd7ab110) acquired here while holding Mutex M2 (0x00007ffccd7ab0e0) in Thread 3120371 (lockinversion):
-@ 00000000004024d0 [unknown]
-@ 0000000000406f4e std::mutex::lock()
-@ 0000000000407250 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
-@ 0000000000402de0 main::{lambda()#2}::operator()() const
-@ 0000000000406d88 void std::_Bind_simple<main::{lambda()#2} ()>::_M_invoke<>(std::_Index_tuple<>)
-@ 0000000000406af3 std::_Bind_simple<main::{lambda()#2} ()>::operator()()
-@ 00000000004068d6 std::thread::_Impl<std::_Bind_simple<main::{lambda()#2} ()> >::_M_run()
-@ 00007f9f9791f4e1 execute_native_thread_routine
-@ 00007f9f9809e7f1 start_thread
-@ 00007f9f9736046d __clone
-
-Mutex M2 (0x00007ffccd7ab0e0) previously acquired by the same Thread 3120371 (lockinversion) here:
-@ 00000000004024d0 [unknown]
-@ 0000000000406f4e std::mutex::lock()
-@ 0000000000407250 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
+Cycle in lock order graph: Mutex M0 (main::static_mutex3 0x0000000000473c60) => Mutex M1 (0x00007fff6d738400) => Mutex M2 (global_mutex1 0x0000000000473be0) => Mutex M3 (global_mutex2 0x0000000000473c20) => Mutex M0 (main::static_mutex3 0x0000000000473c60)
+
+Mutex M1 (0x00007fff6d738400) acquired here while holding Mutex M0 (main::static_mutex3 0x0000000000473c60) in Thread 357250 (lockinversion):
+@ 00000000004024d0 pthread_mutex_lock
+@ 0000000000406dd0 std::mutex::lock()
+@ 00000000004070d2 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
+@ 0000000000402e38 main::{lambda()#3}::operator()() const
+@ 0000000000406ba8 void std::_Bind_simple<main::{lambda()#3} ()>::_M_invoke<>(std::_Index_tuple<>)
+@ 0000000000406951 std::_Bind_simple<main::{lambda()#3} ()>::operator()()
+@ 000000000040673a std::thread::_Impl<std::_Bind_simple<main::{lambda()#3} ()> >::_M_run()
+@ 00007fd4496564e1 execute_native_thread_routine
+@ 00007fd449dd57f1 start_thread
+@ 00007fd44909746d __clone
+
+Mutex M0 (main::static_mutex3 0x0000000000473c60) previously acquired by the same Thread 357250 (lockinversion) here:
+@ 00000000004024d0 pthread_mutex_lock
+@ 0000000000406dd0 std::mutex::lock()
+@ 00000000004070d2 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
+@ 0000000000402e22 main::{lambda()#3}::operator()() const
+@ 0000000000406ba8 void std::_Bind_simple<main::{lambda()#3} ()>::_M_invoke<>(std::_Index_tuple<>)
+@ 0000000000406951 std::_Bind_simple<main::{lambda()#3} ()>::operator()()
+@ 000000000040673a std::thread::_Impl<std::_Bind_simple<main::{lambda()#3} ()> >::_M_run()
+@ 00007fd4496564e1 execute_native_thread_routine
+@ 00007fd449dd57f1 start_thread
+@ 00007fd44909746d __clone
+
+Mutex M2 (global_mutex1 0x0000000000473be0) acquired here while holding Mutex M1 (0x00007fff6d738400) in Thread 357251 (lockinversion):
+@ 00000000004024d0 pthread_mutex_lock
+@ 0000000000406dd0 std::mutex::lock()
+@ 00000000004070d2 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
+@ 0000000000402ea8 main::{lambda()#4}::operator()() const
+@ 0000000000406b46 void std::_Bind_simple<main::{lambda()#4} ()>::_M_invoke<>(std::_Index_tuple<>)
+@ 000000000040692d std::_Bind_simple<main::{lambda()#4} ()>::operator()()
+@ 000000000040671c std::thread::_Impl<std::_Bind_simple<main::{lambda()#4} ()> >::_M_run()
+@ 00007fd4496564e1 execute_native_thread_routine
+@ 00007fd449dd57f1 start_thread
+@ 00007fd44909746d __clone
+
+Mutex M1 (0x00007fff6d738400) previously acquired by the same Thread 357251 (lockinversion) here:
+@ 00000000004024d0 pthread_mutex_lock
+@ 0000000000406dd0 std::mutex::lock()
+@ 00000000004070d2 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
+@ 0000000000402e97 main::{lambda()#4}::operator()() const
+@ 0000000000406b46 void std::_Bind_simple<main::{lambda()#4} ()>::_M_invoke<>(std::_Index_tuple<>)
+@ 000000000040692d std::_Bind_simple<main::{lambda()#4} ()>::operator()()
+@ 000000000040671c std::thread::_Impl<std::_Bind_simple<main::{lambda()#4} ()> >::_M_run()
+@ 00007fd4496564e1 execute_native_thread_routine
+@ 00007fd449dd57f1 start_thread
+@ 00007fd44909746d __clone
+
+Mutex M3 (global_mutex2 0x0000000000473c20) acquired here while holding Mutex M2 (global_mutex1 0x0000000000473be0) in Thread 357247 (lockinversion):
+@ 00000000004024d0 pthread_mutex_lock
+@ 0000000000406dd0 std::mutex::lock()
+@ 00000000004070d2 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
+@ 0000000000402d5f main::{lambda()#1}::operator()() const
+@ 0000000000406c6c void std::_Bind_simple<main::{lambda()#1} ()>::_M_invoke<>(std::_Index_tuple<>)
+@ 0000000000406999 std::_Bind_simple<main::{lambda()#1} ()>::operator()()
+@ 0000000000406776 std::thread::_Impl<std::_Bind_simple<main::{lambda()#1} ()> >::_M_run()
+@ 00007fd4496564e1 execute_native_thread_routine
+@ 00007fd449dd57f1 start_thread
+@ 00007fd44909746d __clone
+
+Mutex M2 (global_mutex1 0x0000000000473be0) previously acquired by the same Thread 357247 (lockinversion) here:
+@ 00000000004024d0 pthread_mutex_lock
+@ 0000000000406dd0 std::mutex::lock()
+@ 00000000004070d2 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
+@ 0000000000402d4e main::{lambda()#1}::operator()() const
+@ 0000000000406c6c void std::_Bind_simple<main::{lambda()#1} ()>::_M_invoke<>(std::_Index_tuple<>)
+@ 0000000000406999 std::_Bind_simple<main::{lambda()#1} ()>::operator()()
+@ 0000000000406776 std::thread::_Impl<std::_Bind_simple<main::{lambda()#1} ()> >::_M_run()
+@ 00007fd4496564e1 execute_native_thread_routine
+@ 00007fd449dd57f1 start_thread
+@ 00007fd44909746d __clone
+
+Mutex M0 (main::static_mutex3 0x0000000000473c60) acquired here while holding Mutex M3 (global_mutex2 0x0000000000473c20) in Thread 357248 (lockinversion):
+@ 00000000004024d0 pthread_mutex_lock
+@ 0000000000406dd0 std::mutex::lock()
+@ 00000000004070d2 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
@ 0000000000402dc9 main::{lambda()#2}::operator()() const
-@ 0000000000406d88 void std::_Bind_simple<main::{lambda()#2} ()>::_M_invoke<>(std::_Index_tuple<>)
-@ 0000000000406af3 std::_Bind_simple<main::{lambda()#2} ()>::operator()()
-@ 00000000004068d6 std::thread::_Impl<std::_Bind_simple<main::{lambda()#2} ()> >::_M_run()
-@ 00007f9f9791f4e1 execute_native_thread_routine
-@ 00007f9f9809e7f1 start_thread
-@ 00007f9f9736046d __clone
-
-Mutex M0 (0x00007ffccd7ab140) acquired here while holding Mutex M3 (0x00007ffccd7ab110) in Thread 3120372 (lockinversion):
-@ 00000000004024d0 [unknown]
-@ 0000000000406f4e std::mutex::lock()
-@ 0000000000407250 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
-@ 0000000000402e56 main::{lambda()#3}::operator()() const
-@ 0000000000406d26 void std::_Bind_simple<main::{lambda()#3} ()>::_M_invoke<>(std::_Index_tuple<>)
-@ 0000000000406acf std::_Bind_simple<main::{lambda()#3} ()>::operator()()
-@ 00000000004068b8 std::thread::_Impl<std::_Bind_simple<main::{lambda()#3} ()> >::_M_run()
-@ 00007f9f9791f4e1 execute_native_thread_routine
-@ 00007f9f9809e7f1 start_thread
-@ 00007f9f9736046d __clone
-
-Mutex M3 (0x00007ffccd7ab110) previously acquired by the same Thread 3120372 (lockinversion) here:
-@ 00000000004024d0 [unknown]
-@ 0000000000406f4e std::mutex::lock()
-@ 0000000000407250 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
-@ 0000000000402e3f main::{lambda()#3}::operator()() const
-@ 0000000000406d26 void std::_Bind_simple<main::{lambda()#3} ()>::_M_invoke<>(std::_Index_tuple<>)
-@ 0000000000406acf std::_Bind_simple<main::{lambda()#3} ()>::operator()()
-@ 00000000004068b8 std::thread::_Impl<std::_Bind_simple<main::{lambda()#3} ()> >::_M_run()
-@ 00007f9f9791f4e1 execute_native_thread_routine
-@ 00007f9f9809e7f1 start_thread
-@ 00007f9f9736046d __clone
-
-Thread 3120370 created by Thread 3113530 (b'lockinversion') here:
-@ 00007f9f97360431 __clone
-@ 00007f9f9809eef5 pthread_create
-@ 00007f9f97921440 std::thread::_M_start_thread(std::shared_ptr<std::thread::_Impl_base>)
-@ 00000000004033e0 std::thread::thread<main::{lambda()#1}>(main::{lambda()#1}&&)
-@ 0000000000403167 main
-@ 00007f9f972730f6 __libc_start_main
+@ 0000000000406c0a void std::_Bind_simple<main::{lambda()#2} ()>::_M_invoke<>(std::_Index_tuple<>)
+@ 0000000000406975 std::_Bind_simple<main::{lambda()#2} ()>::operator()()
+@ 0000000000406758 std::thread::_Impl<std::_Bind_simple<main::{lambda()#2} ()> >::_M_run()
+@ 00007fd4496564e1 execute_native_thread_routine
+@ 00007fd449dd57f1 start_thread
+@ 00007fd44909746d __clone
+
+Mutex M3 (global_mutex2 0x0000000000473c20) previously acquired by the same Thread 357248 (lockinversion) here:
+@ 00000000004024d0 pthread_mutex_lock
+@ 0000000000406dd0 std::mutex::lock()
+@ 00000000004070d2 std::lock_guard<std::mutex>::lock_guard(std::mutex&)
+@ 0000000000402db8 main::{lambda()#2}::operator()() const
+@ 0000000000406c0a void std::_Bind_simple<main::{lambda()#2} ()>::_M_invoke<>(std::_Index_tuple<>)
+@ 0000000000406975 std::_Bind_simple<main::{lambda()#2} ()>::operator()()
+@ 0000000000406758 std::thread::_Impl<std::_Bind_simple<main::{lambda()#2} ()> >::_M_run()
+@ 00007fd4496564e1 execute_native_thread_routine
+@ 00007fd449dd57f1 start_thread
+@ 00007fd44909746d __clone
+
+Thread 357248 created by Thread 350692 (lockinversion) here:
+@ 00007fd449097431 __clone
+@ 00007fd449dd5ef5 pthread_create
+@ 00007fd449658440 std::thread::_M_start_thread(std::shared_ptr<std::thread::_Impl_base>)
+@ 00000000004033ac std::thread::thread<main::{lambda()#2}>(main::{lambda()#2}&&)
+@ 000000000040308f main
+@ 00007fd448faa0f6 __libc_start_main
@ 0000000000402ad8 [unknown]
-Thread 3120371 created by Thread 3113530 (b'lockinversion') here:
-@ 00007f9f97360431 __clone
-@ 00007f9f9809eef5 pthread_create
-@ 00007f9f97921440 std::thread::_M_start_thread(std::shared_ptr<std::thread::_Impl_base>)
-@ 00000000004034e6 std::thread::thread<main::{lambda()#2}>(main::{lambda()#2}&&)
-@ 000000000040319f main
-@ 00007f9f972730f6 __libc_start_main
+Thread 357250 created by Thread 350692 (lockinversion) here:
+@ 00007fd449097431 __clone
+@ 00007fd449dd5ef5 pthread_create
+@ 00007fd449658440 std::thread::_M_start_thread(std::shared_ptr<std::thread::_Impl_base>)
+@ 00000000004034b2 std::thread::thread<main::{lambda()#3}>(main::{lambda()#3}&&)
+@ 00000000004030b9 main
+@ 00007fd448faa0f6 __libc_start_main
@ 0000000000402ad8 [unknown]
-Thread 3120372 created by Thread 3113530 (b'lockinversion') here:
-@ 00007f9f97360431 __clone
-@ 00007f9f9809eef5 pthread_create
-@ 00007f9f97921440 std::thread::_M_start_thread(std::shared_ptr<std::thread::_Impl_base>)
-@ 00000000004035ec std::thread::thread<main::{lambda()#3}>(main::{lambda()#3}&&)
-@ 00000000004031da main
-@ 00007f9f972730f6 __libc_start_main
+Thread 357251 created by Thread 350692 (lockinversion) here:
+@ 00007fd449097431 __clone
+@ 00007fd449dd5ef5 pthread_create
+@ 00007fd449658440 std::thread::_M_start_thread(std::shared_ptr<std::thread::_Impl_base>)
+@ 00000000004035b8 std::thread::thread<main::{lambda()#4}>(main::{lambda()#4}&&)
+@ 00000000004030e6 main
+@ 00007fd448faa0f6 __libc_start_main
@ 0000000000402ad8 [unknown]
-Thread 3120373 created by Thread 3113530 (b'lockinversion') here:
-@ 00007f9f97360431 __clone
-@ 00007f9f9809eef5 pthread_create
-@ 00007f9f97921440 std::thread::_M_start_thread(std::shared_ptr<std::thread::_Impl_base>)
-@ 00000000004036f2 std::thread::thread<main::{lambda()#4}>(main::{lambda()#4}&&)
-@ 0000000000403215 main
-@ 00007f9f972730f6 __libc_start_main
+Thread 357247 created by Thread 350692 (lockinversion) here:
+@ 00007fd449097431 __clone
+@ 00007fd449dd5ef5 pthread_create
+@ 00007fd449658440 std::thread::_M_start_thread(std::shared_ptr<std::thread::_Impl_base>)
+@ 00000000004032a6 std::thread::thread<main::{lambda()#1}>(main::{lambda()#1}&&)
+@ 0000000000403070 main
+@ 00007fd448faa0f6 __libc_start_main
@ 0000000000402ad8 [unknown]
-Nodes: 6, Edges: 5, Looking for cycle took 0.009499 seconds
-
This is output from a process that has a potential deadlock involving 4 mutexes
and 4 threads:
-- Thread 3120373 acquired M1 while holding M0 (edge M0 -> M1)
-- Thread 3120370 acquired M2 while holding M1 (edge M1 -> M2)
-- Thread 3120371 acquired M3 while holding M2 (edge M2 -> M3)
-- Thread 3120372 acquired M0 while holding M3 (edge M3 -> M0)
+- Thread 357250 acquired M1 while holding M0 (edge M0 -> M1)
+- Thread 357251 acquired M2 while holding M1 (edge M1 -> M2)
+- Thread 357247 acquired M3 while holding M2 (edge M2 -> M3)
+- Thread 357248 acquired M0 while holding M3 (edge M3 -> M0)
This is the C++ program that generated the output above:
```c++
-#include <sys/types.h>
-#include <unistd.h>
#include <chrono>
#include <iostream>
#include <mutex>
#include <thread>
-int main(void) {
- std::mutex m1;
- std::mutex m2;
- std::mutex m3;
- std::mutex m4;
+std::mutex global_mutex1;
+std::mutex global_mutex2;
- std::cout << "&m1: " << (void*)&m1 << std::endl;
- std::cout << "&m2: " << (void*)&m2 << std::endl;
- std::cout << "&m3: " << (void*)&m3 << std::endl;
- std::cout << "&m4: " << (void*)&m4 << std::endl;
+int main(void) {
+ static std::mutex static_mutex3;
+ std::mutex local_mutex4;
- std::cout << "pid: " << getpid() << std::endl;
std::cout << "sleeping for a bit to allow trace to attach..." << std::endl;
std::this_thread::sleep_for(std::chrono::seconds(10));
std::cout << "starting program..." << std::endl;
- auto t1 = std::thread([&m1, &m2] {
- std::lock_guard<std::mutex> g1(m1);
- std::lock_guard<std::mutex> g2(m2);
+ auto t1 = std::thread([] {
+ std::lock_guard<std::mutex> g1(global_mutex1);
+ std::lock_guard<std::mutex> g2(global_mutex2);
});
t1.join();
- auto t2 = std::thread([&m2, &m3] {
- std::lock_guard<std::mutex> g2(m2);
- std::lock_guard<std::mutex> g3(m3);
+ auto t2 = std::thread([] {
+ std::lock_guard<std::mutex> g2(global_mutex2);
+ std::lock_guard<std::mutex> g3(static_mutex3);
});
t2.join();
- auto t3 = std::thread([&m3, &m4] {
- std::lock_guard<std::mutex> g3(m3);
- std::lock_guard<std::mutex> g4(m4);
+ auto t3 = std::thread([&local_mutex4] {
+ std::lock_guard<std::mutex> g3(static_mutex3);
+ std::lock_guard<std::mutex> g4(local_mutex4);
});
t3.join();
- auto t4 = std::thread([&m1, &m4] {
- std::lock_guard<std::mutex> g4(m4);
- std::lock_guard<std::mutex> g1(m1);
+ auto t4 = std::thread([&local_mutex4] {
+ std::lock_guard<std::mutex> g4(local_mutex4);
+ std::lock_guard<std::mutex> g1(global_mutex1);
});
t4.join();
Note that an actual deadlock did not occur, although this mutex lock ordering
creates the possibility of a deadlock, and this is a hint to the programmer to
-reconsider the lock ordering.
+reconsider the lock ordering. If the mutexes are global or static and debug
+symbols are enabled, the output will contain the mutex symbol name. The output
+uses a similar format as ThreadSanitizer
+(https://github.com/google/sanitizers/wiki/ThreadSanitizerDeadlockDetector).
-# ./deadlock_detector.py /path/to/program $(pidof program) --dump-graph graph.json
+# ./deadlock_detector.py $(pidof program) --dump-graph graph.json --verbose
Tracing... Hit Ctrl-C to end.
-Nodes: 0, Edges: 0, Looking for cycle took 0.000062 seconds
-Nodes: 0, Edges: 0, Looking for cycle took 0.000066 seconds
-Nodes: 0, Edges: 0, Looking for cycle took 0.000065 seconds
-Nodes: 0, Edges: 0, Looking for cycle took 0.000053 seconds
-Nodes: 102, Edges: 4936, Looking for cycle took 0.155751 seconds
-Nodes: 102, Edges: 4951, Looking for cycle took 0.141393 seconds
-Nodes: 102, Edges: 4951, Looking for cycle took 0.119585 seconds
-Nodes: 102, Edges: 4951, Looking for cycle took 0.118088 seconds
+Mutexes: 0, Edges: 0
+Mutexes: 532, Edges: 411
+Mutexes: 735, Edges: 675
+Mutexes: 1118, Edges: 1278
+Mutexes: 1666, Edges: 2185
+Mutexes: 2056, Edges: 2694
+Mutexes: 2245, Edges: 2906
+Mutexes: 2656, Edges: 3479
+Mutexes: 2813, Edges: 3785
^C
If the program does not find a deadlock, it will keep running until you hit
-Ctrl-C. It will also dump statistics about the number of nodes and edges in
-the mutex wait graph. If you want to serialize the graph to analyze it later,
-you can pass the `--dump-graph FILE` flag, and the program will serialize
-the graph in json format.
+Ctrl-C. If you pass the `--verbose` flag, the program will also dump statistics
+about the number of mutexes and edges in the mutex wait graph. If you want to
+serialize the graph to analyze it later, you can pass the `--dump-graph FILE`
+flag, and the program will serialize the graph in json.
-# ./deadlock_detector.py /path/to/program $(pidof program) --lock-symbols custom_mutex1_lock,custom_mutex2_lock --unlock_symbols custom_mutex1_unlock,custom_mutex2_unlock
+# ./deadlock_detector.py $(pidof program) --lock-symbols custom_mutex1_lock,custom_mutex2_lock --unlock_symbols custom_mutex1_unlock,custom_mutex2_unlock --verbose
Tracing... Hit Ctrl-C to end.
-Nodes: 0, Edges: 0, Looking for cycle took 0.000062 seconds
-Nodes: 0, Edges: 0, Looking for cycle took 0.000066 seconds
-Nodes: 0, Edges: 0, Looking for cycle took 0.000065 seconds
-Nodes: 0, Edges: 0, Looking for cycle took 0.000053 seconds
-Nodes: 102, Edges: 4936, Looking for cycle took 0.155751 seconds
-Nodes: 102, Edges: 4951, Looking for cycle took 0.141393 seconds
-Nodes: 102, Edges: 4951, Looking for cycle took 0.119585 seconds
-Nodes: 102, Edges: 4951, Looking for cycle took 0.118088 seconds
+Mutexes: 0, Edges: 0
+Mutexes: 532, Edges: 411
+Mutexes: 735, Edges: 675
+Mutexes: 1118, Edges: 1278
+Mutexes: 1666, Edges: 2185
+Mutexes: 2056, Edges: 2694
+Mutexes: 2245, Edges: 2906
+Mutexes: 2656, Edges: 3479
+Mutexes: 2813, Edges: 3785
^C
If your program is using custom mutexes and not pthread mutexes, you can use
in false positives.
+# ./deadlock_detector.py $(pidof program) --binary /path/to/my/program
+
+Tracing... Hit Ctrl-C to end.
+^C
+
+You can optionally pass the path to the binary for your process.
+
+By default, the path to the binary is not required and all the user needs to
+provide is the pid of the process. However, on older kernels without this patch
+("uprobe: Find last occurrence of ':' when parsing uprobe PATH:OFFSET",
+https://lkml.org/lkml/2017/1/13/585), binaries that contain `:` in the path
+cannot be attached with uprobes. As a workaround, we can create a symlink
+to the binary, and provide the symlink name instead with the `--binary` option.
+
+
USAGE message:
# ./deadlock_detector.py -h
-usage: deadlock_detector.py [-h] [--dump-graph DUMP_GRAPH]
- [--lock-symbols LOCK_SYMBOLS]
+usage: deadlock_detector.py [-h] [--binary BINARY] [--dump-graph DUMP_GRAPH]
+ [--verbose] [--lock-symbols LOCK_SYMBOLS]
[--unlock-symbols UNLOCK_SYMBOLS]
- binary pid
+ pid
Detect potential deadlocks (lock inversions) in a running binary. Must be run
as root.
positional arguments:
- binary Absolute path to binary
pid Pid to trace
optional arguments:
-h, --help show this help message and exit
+ --binary BINARY If set, use this as the path to the binary for the
+ process.
--dump-graph DUMP_GRAPH
If set, this will dump the mutex graph to the
specified file.
+ --verbose Print statistics about the mutex wait graph.
--lock-symbols LOCK_SYMBOLS
Comma-separated list of lock symbols to trace. Default
is pthread_mutex_lock
projects / platform / upstream / bcc.git / commitdiff