add bpf_map_lookup_and_delete_batch in bcc (iovisor#3234)

* add bpf_map_lookup_and_delete_batch in bcc
* add test_map_batch_ops.py to test batch lookup and delete on map
* add items_lookup_and_delete_batch() in the reference guide

docs/reference_guide.md

@@ -102,12 +102,13 @@ This guide is incomplete. If something feels missing, check the bcc and kernel s
  - [3. items()](#3-items)
  - [4. values()](#4-values)
  - [5. clear()](#5-clear)
- - [6. print_log2_hist()](#6-print_log2_hist)
- - [7. print_linear_hist()](#7-print_linear_hist)
- - [8. open_ring_buffer()](#8-open_ring_buffer)
- - [9. push()](#9-push)
- - [10. pop()](#10-pop)
- - [11. peek()](#11-peek)
+ - [6. items_lookup_and_delete_batch()](#6-items_lookup_and_delete_batch)
+ - [7. print_log2_hist()](#7-print_log2_hist)
+ - [8. print_linear_hist()](#8-print_linear_hist)
+ - [9. open_ring_buffer()](#9-open_ring_buffer)
+ - [10. push()](#10-push)
+ - [11. pop()](#11-pop)
+ - [12. peek()](#12-peek)
  - [Helpers](#helpers)
  - [1. ksym()](#1-ksym)
  - [2. ksymname()](#2-ksymname)
@@ -1912,7 +1913,25 @@ Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=clear+path%3Aexamples+language%3Apython&type=Code),
 [search /tools](https://github.com/iovisor/bcc/search?q=clear+path%3Atools+language%3Apython&type=Code)
 
-### 6. print_log2_hist()
+### 6. items_lookup_and_delete_batch()
+
+Syntax: ```table.items_lookup_and_delete_batch()```
+
+Returns an array of the keys in a table with a single call to BPF syscall. This can be used with BPF_HASH maps to fetch, and iterate, over the keys. It also clears the table: deletes all entries.
+You should rather use table.items_lookup_and_delete_batch() than table.items() followed by table.clear().
+
+Example:
+
+```Python
+# print call rate per second:
+print("%9s-%9s-%8s-%9s" % ("PID", "COMM", "fname", "counter"))
+while True:
+ for k, v in sorted(b['map'].items_lookup_and_delete_batch(), key=lambda kv: (kv[0]).pid):
+ print("%9s-%9s-%8s-%9d" % (k.pid, k.comm, k.fname, v.counter))
+ sleep(1)
+```
+
+### 7. print_log2_hist()
 
 Syntax: ```table.print_log2_hist(val_type="value", section_header="Bucket ptr", section_print_fn=None)```
 
@@ -1963,7 +1982,7 @@ Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=print_log2_hist+path%3Aexamples+language%3Apython&type=Code),
 [search /tools](https://github.com/iovisor/bcc/search?q=print_log2_hist+path%3Atools+language%3Apython&type=Code)
 
-### 6. print_linear_hist()
+### 8. print_linear_hist()
 
 Syntax: ```table.print_linear_hist(val_type="value", section_header="Bucket ptr", section_print_fn=None)```
 
@@ -2022,7 +2041,7 @@ Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=print_linear_hist+path%3Aexamples+language%3Apython&type=Code),
 [search /tools](https://github.com/iovisor/bcc/search?q=print_linear_hist+path%3Atools+language%3Apython&type=Code)
 
-### 8. open_ring_buffer()
+### 9. open_ring_buffer()
 
 Syntax: ```table.open_ring_buffer(callback, ctx=None)```
 
@@ -2084,7 +2103,7 @@ def print_event(ctx, data, size):
 Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=open_ring_buffer+path%3Aexamples+language%3Apython&type=Code),
 
-### 9. push()
+### 10. push()
 
 Syntax: ```table.push(leaf, flags=0)```
 
@@ -2094,7 +2113,7 @@ Passing QueueStack.BPF_EXIST as a flag causes the Queue or Stack to discard the
 Examples in situ:
 [search /tests](https://github.com/iovisor/bcc/search?q=push+path%3Atests+language%3Apython&type=Code),
 
-### 10. pop()
+### 11. pop()
 
 Syntax: ```leaf = table.pop()```
 
@@ -2105,7 +2124,7 @@ Raises a KeyError exception if the operation does not succeed.
 Examples in situ:
 [search /tests](https://github.com/iovisor/bcc/search?q=pop+path%3Atests+language%3Apython&type=Code),
 
-### 11. peek()
+### 12. peek()
 
 Syntax: ```leaf = table.peek()```
 

src/cc/libbpf.c

@@ -356,6 +356,13 @@ int bpf_lookup_and_delete(int fd, void *key, void *value)
  return bpf_map_lookup_and_delete_elem(fd, key, value);
 }
 
+int bpf_lookup_and_delete_batch(int fd, __u32 *in_batch, __u32 *out_batch, void *keys,
+ void *values, __u32 *count)
+{
+ return bpf_map_lookup_and_delete_batch(fd, in_batch, out_batch, keys, values,
+ count, NULL);
+}
+
 int bpf_get_first_key(int fd, void *key, size_t key_size)
 {
  int i, res;

src/python/bcc/libbcc.py

@@ -83,6 +83,9 @@
  ct.c_ulonglong]
 lib.bpf_delete_elem.restype = ct.c_int
 lib.bpf_delete_elem.argtypes = [ct.c_int, ct.c_void_p]
+lib.bpf_lookup_and_delete_batch.restype = ct.c_int
+lib.bpf_lookup_and_delete_batch.argtypes = [ct.c_int, ct.POINTER(ct.c_uint32),
+ ct.POINTER(ct.c_uint32),ct.c_void_p, ct.c_void_p, ct.c_void_p]
 lib.bpf_open_raw_sock.restype = ct.c_int
 lib.bpf_open_raw_sock.argtypes = [ct.c_char_p]
 lib.bpf_attach_socket.restype = ct.c_int

src/python/bcc/table.py

@@ -397,6 +397,31 @@ def clear(self):
  for k in self.keys():
  self.__delitem__(k)
 
+ def items_lookup_and_delete_batch(self):
+ # batch size is set to the maximum
+ batch_size = self.max_entries
+ out_batch = ct.c_uint32(0)
+ keys = (type(self.Key()) * batch_size)()
+ values = (type(self.Leaf()) * batch_size)()
+ count = ct.c_uint32(batch_size)
+ res = lib.bpf_lookup_and_delete_batch(self.map_fd,
+ None,
+ ct.byref(out_batch),
+ ct.byref(keys),
+ ct.byref(values),
+ ct.byref(count)
+ )
+
+ errcode = ct.get_errno()
+ if (errcode == errno.EINVAL):
+ raise Exception("BPF_MAP_LOOKUP_AND_DELETE_BATCH is invalid.")
+
+ if (res != 0 and errcode != errno.ENOENT):
+ raise Exception("BPF_MAP_LOOKUP_AND_DELETE_BATCH has failed")
+
+ for i in range(0, count.value):
+ yield (keys[i], values[i])
+
  def zero(self):
  # Even though this is not very efficient, we grab the entire list of
  # keys before enumerating it. This helps avoid a potential race where
@@ -584,6 +609,8 @@ def print_linear_hist(self, val_type="value", section_header="Bucket ptr",
 class HashTable(TableBase):
  def __init__(self, *args, **kwargs):
  super(HashTable, self).__init__(*args, **kwargs)
+ self.max_entries = int(lib.bpf_table_max_entries_id(self.bpf.module,
+ self.map_id))
 
  def __len__(self):
  i = 0

tests/python/CMakeLists.txt

@@ -93,3 +93,5 @@ add_test(NAME py_ringbuf WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
  COMMAND ${TEST_WRAPPER} py_ringbuf sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_ringbuf.py)
 add_test(NAME py_queuestack WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
  COMMAND ${TEST_WRAPPER} py_queuestack sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_queuestack.py)
+add_test(NAME py_test_map_batch_ops WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+ COMMAND ${TEST_WRAPPER} py_test_map_batch_ops sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_map_batch_ops.py)

tests/python/test_map_batch_ops.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+#
+# USAGE: test_map_batch_ops.py
+#
+# Copyright (c) Emilien Gobillot
+# Licensed under the Apache License, Version 2.0 (the "License")
+
+from __future__ import print_function
+from bcc import BPF
+import distutils.version
+from unittest import main, skipUnless, TestCase
+import ctypes as ct
+import os
+
+
+def kernel_version_ge(major, minor):
+ # True if running kernel is >= X.Y
+ version = distutils.version.LooseVersion(os.uname()[2]).version
+ if version[0] > major:
+ return True
+ if version[0] < major:
+ return False
+ if minor and version[1] < minor:
+ return False
+ return True
+
+
+@skipUnless(kernel_version_ge(5, 6), "requires kernel >= 5.6")
+class TestMapBatch(TestCase):
+ def test_lookup_and_delete_batch(self):
+ b = BPF(text="""BPF_HASH(map, int, int, 1024);""")
+ hmap = b["map"]
+ for i in range(0, 1024):
+ hmap[ct.c_int(i)] = ct.c_int(i)
+
+ # check the lookup
+ i = 0
+ for k, v in sorted(hmap.items_lookup_and_delete_batch()):
+ self.assertEqual(k, i)
+ self.assertEqual(v, i)
+ i += 1
+ # and check the delete has workd, i.e map is empty
+ count = sum(1 for _ in hmap.items_lookup_and_delete_batch())
+ self.assertEqual(count, 0)
+
+
+if __name__ == "__main__":
+ main()