Use local key/leaf definition instead of bpf_tunnel_key

Even after fixing the nested union issue in the preceding commits, this
example was failing. The reason was that the kernel was filling in all
bits of the bpf_tunnel_key, which in recent kernels includes non-zero
fields such as ttl. That non-zero field would break the lookup in the
tunkey2if table, which was populated assuming all extra bytes would be
zero.

Fix it by defining a simpler key structure, and copy the required fields
out from the local variable.

Fixes: iovisor#510

examples/networking/distributed_bridge/tunnel_mesh.c

@@ -7,24 +7,31 @@ struct config {
 };
 BPF_TABLE("hash", int, struct config, conf, 1);
 
-BPF_TABLE("hash", struct bpf_tunnel_key, int, tunkey2if, 1024);
+struct tunnel_key {
+ u32 tunnel_id;
+ u32 remote_ipv4;
+};
+BPF_TABLE("hash", struct tunnel_key, int, tunkey2if, 1024);
 
-BPF_TABLE("hash", int, struct bpf_tunnel_key, if2tunkey, 1024);
+BPF_TABLE("hash", int, struct tunnel_key, if2tunkey, 1024);
 
 // Handle packets from the encap device, demux into the dest tenant
 int handle_ingress(struct __sk_buff *skb) {
  struct bpf_tunnel_key tkey = {};
+ struct tunnel_key key;
  bpf_skb_get_tunnel_key(skb, &tkey, sizeof(tkey), 0);
 
- int *ifindex = tunkey2if.lookup(&tkey);
+ key.tunnel_id = tkey.tunnel_id;
+ key.remote_ipv4 = tkey.remote_ipv4;
+ int *ifindex = tunkey2if.lookup(&key);
  if (ifindex) {
  //bpf_trace_printk("ingress tunnel_id=%d remote_ip=%08x ifindex=%d\n",
- // tkey.tunnel_id, tkey.remote_ipv4, *ifindex);
+ // key.tunnel_id, key.remote_ipv4, *ifindex);
  // mark from external
  skb->tc_index = 1;
  bpf_clone_redirect(skb, *ifindex, 1/*ingress*/);
  } else {
- bpf_trace_printk("ingress invalid tunnel_id=%d\n", tkey.tunnel_id);
+ bpf_trace_printk("ingress invalid tunnel_id=%d\n", key.tunnel_id);
  }
 
  return 1;
@@ -33,7 +40,8 @@ int handle_ingress(struct __sk_buff *skb) {
 // Handle packets from the tenant, mux into the encap device
 int handle_egress(struct __sk_buff *skb) {
  int ifindex = skb->ifindex;
- struct bpf_tunnel_key *tkey_p, tkey = {};
+ struct bpf_tunnel_key tkey = {};
+ struct tunnel_key *key_p;
  int one = 1;
  struct config *cfg = conf.lookup(&one);
 
@@ -45,10 +53,10 @@ int handle_egress(struct __sk_buff *skb) {
  return 1;
  }
 
- tkey_p = if2tunkey.lookup(&ifindex);
- if (tkey_p) {
- tkey.tunnel_id = tkey_p->tunnel_id;
- tkey.remote_ipv4 = tkey_p->remote_ipv4;
+ key_p = if2tunkey.lookup(&ifindex);
+ if (key_p) {
+ tkey.tunnel_id = key_p->tunnel_id;
+ tkey.remote_ipv4 = key_p->remote_ipv4;
  bpf_skb_set_tunnel_key(skb, &tkey, sizeof(tkey), 0);
  bpf_clone_redirect(skb, cfg->tunnel_ifindex, 0/*egress*/);
  }

examples/networking/distributed_bridge/tunnel_mesh.py

@@ -47,7 +47,7 @@ def run():
  ifc_gc.append(vx.ifname)
  else:
  with ipdb.create(ifname="vxlan0", kind="vxlan", vxlan_id=0,
- vxlan_link=ifc, vxlan_port=htons(4789),
+ vxlan_link=ifc, vxlan_port=4789,
  vxlan_collect_metadata=True,
  vxlan_learning=False) as vx:
  vx.up()
@@ -66,12 +66,14 @@ def run():
  if i != host_id:
  v = ipdb.create(ifname="dummy%d%d" % (j , i), kind="dummy").up().commit()
  ipaddr = "172.16.1.%d" % (100 + i)
- tunkey2if_key = tunkey2if.Key(vni, IPAddress(ipaddr))
+ tunkey2if_key = tunkey2if.Key(vni)
+ tunkey2if_key.remote_ipv4 = IPAddress(ipaddr)
  tunkey2if_leaf = tunkey2if.Leaf(v.index)
  tunkey2if[tunkey2if_key] = tunkey2if_leaf
 
  if2tunkey_key = if2tunkey.Key(v.index)
- if2tunkey_leaf = if2tunkey.Leaf(vni, IPAddress(ipaddr))
+ if2tunkey_leaf = if2tunkey.Leaf(vni)
+ if2tunkey_leaf.remote_ipv4 = IPAddress(ipaddr)
  if2tunkey[if2tunkey_key] = if2tunkey_leaf
 
  ipr.tc("add", "sfq", v.index, "1:")
@@ -124,7 +126,7 @@ def run():
  while retry < 0:
  check = Popen(["ip", "addr", "show", "br%d" % j], stdout=PIPE, stderr=PIPE)
  out = check.stdout.read()
- checkip = "99.1.%d" % j
+ checkip = b"99.1.%d" % j
  retry = out.find(checkip)
 
 try: