Skip to content

Commit

Permalink
[test][python] Clear deprecation warnings
Browse files Browse the repository at this point in the history
Strings were used in place of byte-arrays in many places.
While this worked, this is causing a lot of Deprecation warnings.

Eventually, this deprecation warning will become and error, so we may as well
get ahead of time, but on a more practical side, this is causing a lot of
noise in test run logs, making it more complicated to troubleshoot potential failures...

This diff cleans this for the most part.
Other smaller fix will be added on top to address other deprecation warnings.

Before:

```
$ docker run -ti \
                    --privileged \
                    --network=host \
                    --pid=host \
                    -v $(pwd):/bcc \
                    -v /sys/kernel/debug:/sys/kernel/debug:rw \
                    -v /lib/modules:/lib/modules:ro \
                    -v /usr/src:/usr/src:ro \
                    -e CTEST_OUTPUT_ON_FAILURE=1 \
                    bcc-docker-f34 /bin/bash -c \
                    'cd /bcc/build && \
                     make test PYTHON_TEST_LOGFILE=critical.before.log ARGS=-V' 2> /tmp/err.before.log > /tmp/out.before.log
```

Resulted in critical.before.log: https://gist.github.com/chantra/55b577cbccff8ef77af427342fac6dce
out.before.log: https://gist.github.com/chantra/35889f6d089ed5e06b3bfa920e066e0b
err.before.log (empty)

After:
```
docker run -ti \
                    --privileged \
                    --network=host \
                    --pid=host \
                    -v $(pwd):/bcc \
                    -v /sys/kernel/debug:/sys/kernel/debug:rw \
                    -v /lib/modules:/lib/modules:ro \
                    -v /usr/src:/usr/src:ro \
                    -e CTEST_OUTPUT_ON_FAILURE=1 \
                    bcc-docker-f34 /bin/bash -c \
                    'cd /bcc/build && \
                     make test PYTHON_TEST_LOGFILE=critical.after.log ARGS=-V' 2> /tmp/err.after.log > /tmp/out.after.log
```
critical.after.log: https://gist.github.com/chantra/04955b8cfdfb80cc1c23720f659b7b46
out.after.log: https://gist.github.com/chantra/968b7324fbb10490fcb4b3a1d1bf5a28
err.after.log (empty)

Count of DeprecationWarning:
```
$ grep -c Deprecation /tmp/out.*.log
/tmp/out.after.log:0
/tmp/out.before.log:337
```

Diff between critical.{before,after}.llog

https://gist.github.com/chantra/3a890e90aac160f1dbcb6a90ae8fe33d

e.g `test_brb2` seem to have actually failed later then before this diff.
  • Loading branch information
chantra authored and yonghong-song committed Sep 6, 2022
1 parent efee317 commit 3438ec3
Show file tree
Hide file tree
Showing 36 changed files with 429 additions and 429 deletions.
Empty file modified examples/networking/simulation.py
100644 → 100755
Empty file.
2 changes: 1 addition & 1 deletion src/python/bcc/usdt.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ def attach_uprobes(self, bpf, attach_usdt_ignore_pid):
for (binpath, fn_name, addr, pid) in probes:
if attach_usdt_ignore_pid:
pid = -1
bpf.attach_uprobe(name=binpath.decode(), fn_name=fn_name.decode(),
bpf.attach_uprobe(name=binpath, fn_name=fn_name,
addr=addr, pid=pid)

def enumerate_active_probes(self):
Expand Down
32 changes: 16 additions & 16 deletions tests/python/test_array.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@

class TestArray(TestCase):
def test_simple(self):
b = BPF(text="""BPF_ARRAY(table1, u64, 128);""")
t1 = b["table1"]
b = BPF(text=b"""BPF_ARRAY(table1, u64, 128);""")
t1 = b[b"table1"]
t1[ct.c_int(0)] = ct.c_ulonglong(100)
t1[ct.c_int(127)] = ct.c_ulonglong(1000)
for i, v in t1.items():
Expand All @@ -24,8 +24,8 @@ def test_simple(self):
self.assertEqual(len(t1), 128)

def test_native_type(self):
b = BPF(text="""BPF_ARRAY(table1, u64, 128);""")
t1 = b["table1"]
b = BPF(text=b"""BPF_ARRAY(table1, u64, 128);""")
t1 = b[b"table1"]
t1[0] = ct.c_ulonglong(100)
t1[-2] = ct.c_ulonglong(37)
t1[127] = ct.c_ulonglong(1000)
Expand All @@ -52,7 +52,7 @@ def cb(cpu, data, size):
def lost_cb(lost):
self.assertGreater(lost, 0)

text = """
text = b"""
BPF_PERF_OUTPUT(events);
int do_sys_nanosleep(void *ctx) {
struct {
Expand All @@ -63,11 +63,11 @@ def lost_cb(lost):
}
"""
b = BPF(text=text)
b.attach_kprobe(event=b.get_syscall_fnname("nanosleep"),
fn_name="do_sys_nanosleep")
b.attach_kprobe(event=b.get_syscall_fnname("clock_nanosleep"),
fn_name="do_sys_nanosleep")
b["events"].open_perf_buffer(cb, lost_cb=lost_cb)
b.attach_kprobe(event=b.get_syscall_fnname(b"nanosleep"),
fn_name=b"do_sys_nanosleep")
b.attach_kprobe(event=b.get_syscall_fnname(b"clock_nanosleep"),
fn_name=b"do_sys_nanosleep")
b[b"events"].open_perf_buffer(cb, lost_cb=lost_cb)
subprocess.call(['sleep', '0.1'])
b.perf_buffer_poll()
self.assertGreater(self.counter, 0)
Expand All @@ -87,7 +87,7 @@ def cb(cpu, data, size):
def lost_cb(lost):
self.assertGreater(lost, 0)

text = """
text = b"""
BPF_PERF_OUTPUT(events);
int do_sys_nanosleep(void *ctx) {
struct {
Expand All @@ -98,11 +98,11 @@ def lost_cb(lost):
}
"""
b = BPF(text=text)
b.attach_kprobe(event=b.get_syscall_fnname("nanosleep"),
fn_name="do_sys_nanosleep")
b.attach_kprobe(event=b.get_syscall_fnname("clock_nanosleep"),
fn_name="do_sys_nanosleep")
b["events"].open_perf_buffer(cb, lost_cb=lost_cb)
b.attach_kprobe(event=b.get_syscall_fnname(b"nanosleep"),
fn_name=b"do_sys_nanosleep")
b.attach_kprobe(event=b.get_syscall_fnname(b"clock_nanosleep"),
fn_name=b"do_sys_nanosleep")
b[b"events"].open_perf_buffer(cb, lost_cb=lost_cb)
online_cpus = get_online_cpus()
for cpu in online_cpus:
subprocess.call(['taskset', '-c', str(cpu), 'sleep', '0.1'])
Expand Down
12 changes: 6 additions & 6 deletions tests/python/test_attach_perf_event.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ class TestPerfAttachRaw(unittest.TestCase):
@unittest.skipUnless(kernel_version_ge(4,9), "requires kernel >= 4.9")
def test_attach_raw_event_powerpc(self):
# on PowerPC, 'addr' is always written to; for x86 see _x86 version of test
bpf_text="""
bpf_text=b"""
#include <linux/perf_event.h>
struct key_t {
int cpu;
Expand Down Expand Up @@ -56,7 +56,7 @@ def test_attach_raw_event_powerpc(self):
event_attr.sample_period = 1000000
event_attr.sample_type = PerfEventSampleFormat.ADDR
event_attr.exclude_kernel = 1
b.attach_perf_event_raw(attr=event_attr, fn_name="on_sample_hit", pid=-1, cpu=-1)
b.attach_perf_event_raw(attr=event_attr, fn_name=b"on_sample_hit", pid=-1, cpu=-1)
except Exception:
print("Failed to attach to a raw event. Please check the event attr used")
exit()
Expand All @@ -68,7 +68,7 @@ def test_attach_raw_event_powerpc(self):
@unittest.skipUnless(kernel_version_ge(4,17), "bpf_perf_event_data->addr requires kernel >= 4.17")
def test_attach_raw_event_x86(self):
# on x86, need to set precise_ip in order for perf_events to write to 'addr'
bpf_text="""
bpf_text=b"""
#include <linux/perf_event.h>
struct key_t {
int cpu;
Expand Down Expand Up @@ -102,7 +102,7 @@ def test_attach_raw_event_x86(self):
event_attr.sample_type = PerfEventSampleFormat.ADDR
event_attr.exclude_kernel = 1
event_attr.precise_ip = 2
b.attach_perf_event_raw(attr=event_attr, fn_name="on_sample_hit", pid=-1, cpu=-1)
b.attach_perf_event_raw(attr=event_attr, fn_name=b"on_sample_hit", pid=-1, cpu=-1)
except Exception:
print("Failed to attach to a raw event. Please check the event attr used")
exit()
Expand All @@ -114,7 +114,7 @@ def test_attach_raw_event_x86(self):
# SW perf events should work on GH actions, so expect this to succeed
@unittest.skipUnless(kernel_version_ge(4,17), "bpf_perf_event_data->addr requires kernel >= 4.17")
def test_attach_raw_sw_event(self):
bpf_text="""
bpf_text=b"""
#include <linux/perf_event.h>
struct key_t {
int cpu;
Expand Down Expand Up @@ -147,7 +147,7 @@ def test_attach_raw_sw_event(self):
event_attr.sample_period = 100
event_attr.sample_type = PerfEventSampleFormat.ADDR
event_attr.exclude_kernel = 1
b.attach_perf_event_raw(attr=event_attr, fn_name="on_sample_hit", pid=-1, cpu=-1)
b.attach_perf_event_raw(attr=event_attr, fn_name=b"on_sample_hit", pid=-1, cpu=-1)
except Exception:
print("Failed to attach to a raw event. Please check the event attr used")
exit()
Expand Down
10 changes: 5 additions & 5 deletions tests/python/test_bpf_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@

error_msg = "R0 invalid mem access 'map_value_or_null'\n"

text = """
text = b"""
#include <uapi/linux/ptrace.h>
#include <bcc/proto.h>
BPF_HASH(t1, int, int, 10);
int sim_port(struct __sk_buff *skb) {
int x = 0, *y;
"""
repeat = """
repeat = b"""
y = t1.lookup(&x);
if (!y) return 0;
x = *y;
"""
end = """
end = b"""
y = t1.lookup(&x);
x = *y;
return 0;
Expand All @@ -47,7 +47,7 @@ def tearDown(self):
def test_log_debug(self):
b = BPF(text=text, debug=2)
try:
ingress = b.load_func("sim_port",BPF.SCHED_CLS)
ingress = b.load_func(b"sim_port",BPF.SCHED_CLS)
except Exception:
self.fp.flush()
self.fp.seek(0)
Expand All @@ -57,7 +57,7 @@ def test_log_debug(self):
def test_log_no_debug(self):
b = BPF(text=text, debug=0)
try:
ingress = b.load_func("sim_port",BPF.SCHED_CLS)
ingress = b.load_func(b"sim_port",BPF.SCHED_CLS)
except Exception:
self.fp.flush()
self.fp.seek(0)
Expand Down
30 changes: 15 additions & 15 deletions tests/python/test_brb.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,20 +89,20 @@ def set_default_const(self):
self.vm2_rtr_mask = "200.1.1.0/24"

def get_table(self, b):
self.jump = b.get_table("jump")
self.jump = b.get_table(b"jump")

self.pem_dest = b.get_table("pem_dest")
self.pem_port = b.get_table("pem_port")
self.pem_ifindex = b.get_table("pem_ifindex")
self.pem_stats = b.get_table("pem_stats")
self.pem_dest = b.get_table(b"pem_dest")
self.pem_port = b.get_table(b"pem_port")
self.pem_ifindex = b.get_table(b"pem_ifindex")
self.pem_stats = b.get_table(b"pem_stats")

self.br1_dest = b.get_table("br1_dest")
self.br1_mac = b.get_table("br1_mac")
self.br1_rtr = b.get_table("br1_rtr")
self.br1_dest = b.get_table(b"br1_dest")
self.br1_mac = b.get_table(b"br1_mac")
self.br1_rtr = b.get_table(b"br1_rtr")

self.br2_dest = b.get_table("br2_dest")
self.br2_mac = b.get_table("br2_mac")
self.br2_rtr = b.get_table("br2_rtr")
self.br2_dest = b.get_table(b"br2_dest")
self.br2_mac = b.get_table(b"br2_mac")
self.br2_rtr = b.get_table(b"br2_rtr")

def connect_ports(self, prog_id_pem, prog_id_br, curr_pem_pid, curr_br_pid,
br_dest_map, br_mac_map, ifindex, vm_mac, vm_ip):
Expand Down Expand Up @@ -150,10 +150,10 @@ def config_maps(self):
@mayFail("If the 'iperf', 'netserver' and 'netperf' binaries are unavailable, this is allowed to fail.")
def test_brb(self):
try:
b = BPF(src_file=arg1, debug=0)
self.pem_fn = b.load_func("pem", BPF.SCHED_CLS)
self.br1_fn = b.load_func("br1", BPF.SCHED_CLS)
self.br2_fn = b.load_func("br2", BPF.SCHED_CLS)
b = BPF(src_file=arg1.encode(), debug=0)
self.pem_fn = b.load_func(b"pem", BPF.SCHED_CLS)
self.br1_fn = b.load_func(b"br1", BPF.SCHED_CLS)
self.br2_fn = b.load_func(b"br2", BPF.SCHED_CLS)
self.get_table(b)

# set up the topology
Expand Down
8 changes: 4 additions & 4 deletions tests/python/test_brb2.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,10 @@ def config_maps(self):
@mayFail("This fails on github actions environment, and needs to be fixed")
def test_brb2(self):
try:
b = BPF(src_file=arg1, debug=0)
self.pem_fn = b.load_func("pem", BPF.SCHED_CLS)
self.pem_dest= b.get_table("pem_dest")
self.pem_stats = b.get_table("pem_stats")
b = BPF(src_file=arg1.encode(), debug=0)
self.pem_fn = b.load_func(b"pem", BPF.SCHED_CLS)
self.pem_dest= b.get_table(b"pem_dest")
self.pem_stats = b.get_table(b"pem_stats")

# set up the topology
self.set_default_const()
Expand Down
16 changes: 8 additions & 8 deletions tests/python/test_call1.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,21 +21,21 @@

class TestBPFSocket(TestCase):
def setUp(self):
b = BPF(src_file=arg1, debug=0)
ether_fn = b.load_func("parse_ether", BPF.SCHED_CLS)
arp_fn = b.load_func("parse_arp", BPF.SCHED_CLS)
ip_fn = b.load_func("parse_ip", BPF.SCHED_CLS)
eop_fn = b.load_func("eop", BPF.SCHED_CLS)
b = BPF(src_file=arg1.encode(), debug=0)
ether_fn = b.load_func(b"parse_ether", BPF.SCHED_CLS)
arp_fn = b.load_func(b"parse_arp", BPF.SCHED_CLS)
ip_fn = b.load_func(b"parse_ip", BPF.SCHED_CLS)
eop_fn = b.load_func(b"eop", BPF.SCHED_CLS)
ip = IPRoute()
ifindex = ip.link_lookup(ifname="eth0")[0]
ifindex = ip.link_lookup(ifname=b"eth0")[0]
ip.tc("add", "sfq", ifindex, "1:")
ip.tc("add-filter", "bpf", ifindex, ":1", fd=ether_fn.fd,
name=ether_fn.name, parent="1:", action="ok", classid=1)
self.jump = b.get_table("jump", c_int, c_int)
self.jump = b.get_table(b"jump", c_int, c_int)
self.jump[c_int(S_ARP)] = c_int(arp_fn.fd)
self.jump[c_int(S_IP)] = c_int(ip_fn.fd)
self.jump[c_int(S_EOP)] = c_int(eop_fn.fd)
self.stats = b.get_table("stats", c_int, c_ulonglong)
self.stats = b.get_table(b"stats", c_int, c_ulonglong)

@mayFail("This may fail on github actions environment due to udp packet loss")
def test_jumps(self):
Expand Down
Loading

0 comments on commit 3438ec3

Please sign in to comment.