add bpf_map_lookup_and_delete_batch in bcc (#3234)
authorEmilien Gobillot <49881870+egobillot@users.noreply.github.com>
Mon, 22 Feb 2021 15:45:16 +0000 (16:45 +0100)
committerGitHub <noreply@github.com>
Mon, 22 Feb 2021 15:45:16 +0000 (07:45 -0800)
* add bpf_map_lookup_and_delete_batch in bcc
* add test_map_batch_ops.py to test batch lookup and delete on map
* add items_lookup_and_delete_batch() in the reference guide

docs/reference_guide.md
src/cc/libbpf.c
src/python/bcc/libbcc.py
src/python/bcc/table.py
tests/python/CMakeLists.txt
tests/python/test_map_batch_ops.py [new file with mode: 0755]

index fa7722efcacf83090b096eca269e47ab052fd76f..f430a57395579e8d4e07a41ed30cefcc2ff91bd3 100644 (file)
@@ -102,12 +102,13 @@ This guide is incomplete. If something feels missing, check the bcc and kernel s
         - [3. items()](#3-items)
         - [4. values()](#4-values)
         - [5. clear()](#5-clear)
-        - [6. print_log2_hist()](#6-print_log2_hist)
-        - [7. print_linear_hist()](#7-print_linear_hist)
-        - [8. open_ring_buffer()](#8-open_ring_buffer)
-        - [9. push()](#9-push)
-        - [10. pop()](#10-pop)
-        - [11. peek()](#11-peek)
+        - [6. items_lookup_and_delete_batch()](#6-items_lookup_and_delete_batch)
+        - [7. print_log2_hist()](#7-print_log2_hist)
+        - [8. print_linear_hist()](#8-print_linear_hist)
+        - [9. open_ring_buffer()](#9-open_ring_buffer)
+        - [10. push()](#10-push)
+        - [11. pop()](#11-pop)
+        - [12. peek()](#12-peek)
     - [Helpers](#helpers)
         - [1. ksym()](#1-ksym)
         - [2. ksymname()](#2-ksymname)
@@ -1912,7 +1913,25 @@ Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=clear+path%3Aexamples+language%3Apython&type=Code),
 [search /tools](https://github.com/iovisor/bcc/search?q=clear+path%3Atools+language%3Apython&type=Code)
 
-### 6. print_log2_hist()
+### 6. items_lookup_and_delete_batch()
+
+Syntax: ```table.items_lookup_and_delete_batch()```
+
+Returns an array of the keys in a table with a single call to BPF syscall. This can be used with BPF_HASH maps to fetch, and iterate, over the keys. It also clears the table: deletes all entries.
+You should rather use table.items_lookup_and_delete_batch() than table.items() followed by table.clear().
+
+Example:
+
+```Python
+# print call rate per second:
+print("%9s-%9s-%8s-%9s" % ("PID", "COMM", "fname", "counter"))
+while True:
+    for k, v in sorted(b['map'].items_lookup_and_delete_batch(), key=lambda kv: (kv[0]).pid):
+        print("%9s-%9s-%8s-%9d" % (k.pid, k.comm, k.fname, v.counter))
+    sleep(1)
+```
+
+### 7. print_log2_hist()
 
 Syntax: ```table.print_log2_hist(val_type="value", section_header="Bucket ptr", section_print_fn=None)```
 
@@ -1963,7 +1982,7 @@ Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=print_log2_hist+path%3Aexamples+language%3Apython&type=Code),
 [search /tools](https://github.com/iovisor/bcc/search?q=print_log2_hist+path%3Atools+language%3Apython&type=Code)
 
-### 6. print_linear_hist()
+### 8. print_linear_hist()
 
 Syntax: ```table.print_linear_hist(val_type="value", section_header="Bucket ptr", section_print_fn=None)```
 
@@ -2022,7 +2041,7 @@ Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=print_linear_hist+path%3Aexamples+language%3Apython&type=Code),
 [search /tools](https://github.com/iovisor/bcc/search?q=print_linear_hist+path%3Atools+language%3Apython&type=Code)
 
-### 8. open_ring_buffer()
+### 9. open_ring_buffer()
 
 Syntax: ```table.open_ring_buffer(callback, ctx=None)```
 
@@ -2084,7 +2103,7 @@ def print_event(ctx, data, size):
 Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=open_ring_buffer+path%3Aexamples+language%3Apython&type=Code),
 
-### 9. push()
+### 10. push()
 
 Syntax: ```table.push(leaf, flags=0)```
 
@@ -2094,7 +2113,7 @@ Passing QueueStack.BPF_EXIST as a flag causes the Queue or Stack to discard the
 Examples in situ:
 [search /tests](https://github.com/iovisor/bcc/search?q=push+path%3Atests+language%3Apython&type=Code),
 
-### 10. pop()
+### 11. pop()
 
 Syntax: ```leaf = table.pop()```
 
@@ -2105,7 +2124,7 @@ Raises a KeyError exception if the operation does not succeed.
 Examples in situ:
 [search /tests](https://github.com/iovisor/bcc/search?q=pop+path%3Atests+language%3Apython&type=Code),
 
-### 11. peek()
+### 12. peek()
 
 Syntax: ```leaf = table.peek()```
 
index 7186ad6066c07e4d5451b5167e2481c65ef68d67..f7d6996af068ea578db802e823ecfc5358afeea3 100644 (file)
@@ -356,6 +356,13 @@ int bpf_lookup_and_delete(int fd, void *key, void *value)
   return bpf_map_lookup_and_delete_elem(fd, key, value);
 }
 
+int bpf_lookup_and_delete_batch(int fd, __u32 *in_batch, __u32 *out_batch, void *keys,
+                                void *values, __u32 *count)
+{
+  return bpf_map_lookup_and_delete_batch(fd, in_batch, out_batch, keys, values,
+                                         count, NULL);
+}
+
 int bpf_get_first_key(int fd, void *key, size_t key_size)
 {
   int i, res;
index 86ba5f2dbf80cb58612a770d4ce7195634508a9a..ef1718c64b7a5eb2efdf6532cf30ade66bd8626e 100644 (file)
@@ -83,6 +83,9 @@ lib.bpf_update_elem.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p,
         ct.c_ulonglong]
 lib.bpf_delete_elem.restype = ct.c_int
 lib.bpf_delete_elem.argtypes = [ct.c_int, ct.c_void_p]
+lib.bpf_lookup_and_delete_batch.restype = ct.c_int
+lib.bpf_lookup_and_delete_batch.argtypes = [ct.c_int, ct.POINTER(ct.c_uint32),
+        ct.POINTER(ct.c_uint32),ct.c_void_p, ct.c_void_p, ct.c_void_p]
 lib.bpf_open_raw_sock.restype = ct.c_int
 lib.bpf_open_raw_sock.argtypes = [ct.c_char_p]
 lib.bpf_attach_socket.restype = ct.c_int
index 16c15b8fb75943fd3ce83942b4ece16cf4a43cb9..a6d38bc05ffc3c91d5da8ff36b6a34bf94c96b86 100644 (file)
@@ -397,6 +397,31 @@ class TableBase(MutableMapping):
         for k in self.keys():
             self.__delitem__(k)
 
+    def items_lookup_and_delete_batch(self):
+        # batch size is set to the maximum
+        batch_size = self.max_entries
+        out_batch = ct.c_uint32(0)
+        keys = (type(self.Key()) * batch_size)()
+        values = (type(self.Leaf()) * batch_size)()
+        count = ct.c_uint32(batch_size)
+        res = lib.bpf_lookup_and_delete_batch(self.map_fd,
+                                              None,
+                                              ct.byref(out_batch),
+                                              ct.byref(keys),
+                                              ct.byref(values),
+                                              ct.byref(count)
+                                              )
+
+        errcode = ct.get_errno()
+        if (errcode == errno.EINVAL):
+            raise Exception("BPF_MAP_LOOKUP_AND_DELETE_BATCH is invalid.")
+
+        if (res != 0 and errcode != errno.ENOENT):
+            raise Exception("BPF_MAP_LOOKUP_AND_DELETE_BATCH has failed")
+
+        for i in range(0, count.value):
+            yield (keys[i], values[i])
+
     def zero(self):
         # Even though this is not very efficient, we grab the entire list of
         # keys before enumerating it. This helps avoid a potential race where
@@ -584,6 +609,8 @@ class TableBase(MutableMapping):
 class HashTable(TableBase):
     def __init__(self, *args, **kwargs):
         super(HashTable, self).__init__(*args, **kwargs)
+        self.max_entries = int(lib.bpf_table_max_entries_id(self.bpf.module,
+                                                            self.map_id))
 
     def __len__(self):
         i = 0
index e7ce5c632fec3d45e5af03e314c1c63a75088552..7e60413ba9cbae7c2604d49c3457dedee615e2f9 100644 (file)
@@ -93,3 +93,5 @@ add_test(NAME py_ringbuf WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
   COMMAND ${TEST_WRAPPER} py_ringbuf sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_ringbuf.py)
 add_test(NAME py_queuestack WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
   COMMAND ${TEST_WRAPPER} py_queuestack sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_queuestack.py)
+add_test(NAME py_test_map_batch_ops WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND ${TEST_WRAPPER} py_test_map_batch_ops sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_map_batch_ops.py)
diff --git a/tests/python/test_map_batch_ops.py b/tests/python/test_map_batch_ops.py
new file mode 100755 (executable)
index 0000000..5b25d19
--- /dev/null
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+#
+# USAGE: test_map_batch_ops.py
+#
+# Copyright (c) Emilien Gobillot
+# Licensed under the Apache License, Version 2.0 (the "License")
+
+from __future__ import print_function
+from bcc import BPF
+import distutils.version
+from unittest import main, skipUnless, TestCase
+import ctypes as ct
+import os
+
+
+def kernel_version_ge(major, minor):
+    # True if running kernel is >= X.Y
+    version = distutils.version.LooseVersion(os.uname()[2]).version
+    if version[0] > major:
+        return True
+    if version[0] < major:
+        return False
+    if minor and version[1] < minor:
+        return False
+    return True
+
+
+@skipUnless(kernel_version_ge(5, 6), "requires kernel >= 5.6")
+class TestMapBatch(TestCase):
+    def test_lookup_and_delete_batch(self):
+        b = BPF(text="""BPF_HASH(map, int, int, 1024);""")
+        hmap = b["map"]
+        for i in range(0, 1024):
+            hmap[ct.c_int(i)] = ct.c_int(i)
+
+        # check the lookup
+        i = 0
+        for k, v in sorted(hmap.items_lookup_and_delete_batch()):
+            self.assertEqual(k, i)
+            self.assertEqual(v, i)
+            i += 1
+        # and check the delete has workd, i.e map is empty
+        count = sum(1 for _ in hmap.items_lookup_and_delete_batch())
+        self.assertEqual(count, 0)
+
+
+if __name__ == "__main__":
+    main()