src/lua: LuaJIT BPF compiler, examples, tests (iovisor#652)

this is initial commit of LuaJIT bytecode to BPF
compiler project that enables writing both kernel
and user-part of the code as Lua

examples/lua/kprobe-latency.lua

@@ -0,0 +1,79 @@
+#!/usr/bin/env bcc-lua
+--[[
+Copyright 2016 Marek Vavrusa <[email protected]>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http:https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]
+-- This example program measures latency of block device operations and plots it
+-- in a histogram. It is similar to BPF example:
+-- https://github.com/torvalds/linux/blob/master/samples/bpf/tracex3_kern.c
+local ffi = require('ffi')
+local bpf = require('bpf')
+local S = require('syscall')
+
+-- Shared part of the program
+local bins = 100
+local map = bpf.map('hash', 512, ffi.typeof('uint64_t'), ffi.typeof('uint64_t'))
+local lat_map = bpf.map('array', bins)
+
+-- Kernel-space part of the program
+local trace_start = bpf.kprobe('myprobe:blk_start_request', function (ptregs)
+ map[ptregs.parm1] = time()
+end, false, -1, 0)
+local trace_end = bpf.kprobe('myprobe2:blk_account_io_completion', function (ptregs)
+ -- The lines below are computing index
+ -- using log10(x)*10 = log2(x)*10/log2(10) = log2(x)*3
+ -- index = 29 ~ 1 usec
+ -- index = 59 ~ 1 msec
+ -- index = 89 ~ 1 sec
+ -- index = 99 ~ 10sec or more
+ local delta = time() - map[ptregs.parm1]
+ local index = 3 * math.log2(delta)
+ if index >= bins then
+ index = bins-1
+ end
+ xadd(lat_map[index], 1)
+ return true
+end, false, -1, 0)
+-- User-space part of the program
+pcall(function()
+ local counter = 0
+ local sym = {' ',' ','.','.','*','*','o','o','O','O','#','#'}
+ while true do
+ -- Print header once in a while
+ if counter % 50 == 0 then
+ print('|1us |10us |100us |1ms |10ms |100ms |1s |10s')
+ counter = 0
+ end
+ counter = counter + 1
+ -- Collect all events
+ local hist, events = {}, 0
+ for i=29,bins-1 do
+ local v = tonumber(lat_map[i] or 0)
+ if v > 0 then
+ hist[i] = hist[i] or 0 + v
+ events = events + v
+ end
+ end
+ -- Print histogram symbols based on relative frequency
+ local s = ''
+ for i=29,bins-1 do
+ if hist[i] then
+ local c = math.ceil((hist[i] / (events + 1)) * #sym)
+ s = s .. sym[c]
+ else s = s .. ' ' end
+ end
+ print(s .. string.format(' ; %d events', events))
+ S.sleep(1)
+ end
+end)

examples/lua/kprobe-write.lua

@@ -0,0 +1,35 @@
+#!/usr/bin/env bcc-lua
+--[[
+Copyright 2016 Marek Vavrusa <[email protected]>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http:https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]
+-- Simple tracing example that executes a program on
+-- return from sys_write() and tracks the number of hits
+local ffi = require('ffi')
+local bpf = require('bpf')
+local S = require('syscall')
+
+-- Shared part of the program
+local map = bpf.map('array', 1)
+-- Kernel-space part of the program
+local probe = bpf.kprobe('myprobe:sys_write', function (ptregs)
+ xadd(map[0], 1)
+end, true)
+-- User-space part of the program
+pcall(function()
+ for _ = 1, 10 do
+ print('hits: ', tonumber(map[0]))
+ S.sleep(1)
+ end
+end)

examples/lua/sock-parse-dns.lua

@@ -0,0 +1,56 @@
+#!/usr/bin/env bcc-lua
+--[[
+Copyright 2016 Marek Vavrusa <[email protected]>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http:https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]
+-- Simple parsing example of UDP/DNS that counts frequency of QTYPEs.
+-- It shows how to parse packet variable-length packet structures.
+local ffi = require("ffi")
+local bpf = require("bpf")
+local S = require("syscall")
+
+-- Shared part of the program
+local map = assert(bpf.map('array', 256))
+-- Kernel-space part of the program
+local prog = bpf.socket('lo', function (skb)
+ local ip = pkt.ip -- Accept only UDP messages
+ if ip.proto ~= c.ip.proto_udp then return false end
+ local udp = ip.udp -- Only messages >12 octets (DNS header)
+ if udp.length < 12 then return false end
+ -- Unroll QNAME (up to 2 labels)
+ udp = udp.data + 12
+ local label = udp[0]
+ if label > 0 then
+ udp = udp + label + 1
+ label = udp[0]
+ if label > 0 then
+ udp = udp + label + 1
+ end
+ end
+ -- Track QTYPE (low types)
+ if udp[0] == 0 then
+ local qtype = udp[2] -- Low octet from QTYPE
+ xadd(map[qtype], 1)
+ end
+end)
+-- User-space part of the program
+for _ = 1, 10 do
+ for k,v in map.pairs,map,0 do
+ v = tonumber(v)
+ if v > 0 then
+ print(string.format('TYPE%d: %d', k, v))
+ end
+ end
+ S.sleep(1)
+end

examples/lua/sock-parse-http.lua

@@ -0,0 +1,57 @@
+#!/usr/bin/env bcc-lua
+--[[
+Copyright 2016 Marek Vavrusa <[email protected]>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http:https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]
+-- Simple parsing example of TCP/HTTP that counts frequency of types of requests
+-- and shows more complicated pattern matching constructions and slices.
+-- Rewrite of a BCC example:
+-- https://github.com/iovisor/bcc/blob/master/examples/networking/http_filter/http-parse-simple.c
+local ffi = require("ffi")
+local bpf = require("bpf")
+local S = require("syscall")
+
+-- Shared part of the program
+local map = bpf.map('hash', 64)
+-- Kernel-space part of the program
+local prog = bpf.socket('lo', function (skb)
+ -- Only ingress so we don't count twice on loopback
+ if skb.ingress_ifindex == 0 then return end
+ local data = pkt.ip.tcp.data -- Get TCP protocol dissector
+ -- Continue only if we have 7 bytes of TCP data
+ if data + 7 > skb.len then return end
+ -- Fetch 4 bytes of TCP data and compare
+ local h = data(0, 4)
+ if h == 'HTTP' or h == 'GET ' or
+ h == 'POST' or h == 'PUT ' or 
+ h == 'HEAD' or h == 'DELE' then
+ -- If hash key doesn't exist, create it
+ -- otherwise increment counter
+ local v = map[h]
+ if not v then map[h] = 1
+ else xadd(map[h], 1)
+ end
+ end
+end)
+-- User-space part of the program
+for _ = 1, 10 do
+ local strkey = ffi.new('uint32_t [1]')
+ local s = ''
+ for k,v in map.pairs,map,0 do
+ strkey[0] = bpf.ntoh(k)
+ s = s..string.format('%s %d ', ffi.string(strkey, 4):match '^%s*(.-)%s*$', tonumber(v))
+ end
+ if #s > 0 then print(s..'messages') end
+ S.sleep(1)
+end

examples/lua/sock-proto.lua

@@ -0,0 +1,38 @@
+#!/usr/bin/env bcc-lua
+--[[
+Copyright 2016 Marek Vavrusa <[email protected]>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http:https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]
+-- This program looks at IP, UDP and ICMP packets and
+-- increments counter for each packet of given type seen
+-- Rewrite of https://github.com/torvalds/linux/blob/master/samples/bpf/sock_example.c
+local ffi = require("ffi")
+local bpf = require("bpf")
+local S = require("syscall")
+
+-- Shared part of the program
+local map = bpf.map('hash', 256)
+map[1], map[6], map[17] = 0, 0, 0
+-- Kernel-space part of the program
+bpf.socket('lo', function (skb)
+ local proto = pkt.ip.proto -- Get byte (ip.proto) from frame at [23]
+ xadd(map[proto], 1) -- Atomic `map[proto] += 1`
+end)
+-- User-space part of the program
+for _ = 1, 10 do
+ local icmp, udp, tcp = map[1], map[17], map[6]
+ print(string.format('TCP %d UDP %d ICMP %d packets',
+ tonumber(tcp or 0), tonumber(udp or 0), tonumber(icmp or 0)))
+ S.sleep(1)
+end

examples/lua/sock-protolen.lua

@@ -0,0 +1,38 @@
+#!/usr/bin/env bcc-lua
+--[[
+Copyright 2016 Marek Vavrusa <[email protected]>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http:https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]
+-- This program counts total bytes received per-protocol in 64-bit counters.
+-- The map backend is array in this case to avoid key allocations.
+-- increments counter for each packet of given type seen
+-- Rewrite of https://github.com/torvalds/linux/blob/master/samples/bpf/sock_example.c
+local ffi = require("ffi")
+local bpf = require("bpf")
+local S = require("syscall")
+
+-- Shared part of the program
+local map = bpf.map('array', 256, ffi.typeof('uint32_t'), ffi.typeof('uint64_t'))
+-- Kernel-space part of the program
+bpf.socket('lo', function (skb)
+ local proto = pkt.ip.proto -- Get byte (ip.proto) from frame at [23]
+ xadd(map[proto], skb.len) -- Atomic `map[proto] += <payload length>`
+end)
+-- User-space part of the program
+for _ = 1, 10 do
+ local icmp, udp, tcp = map[1], map[17], map[6]
+ print(string.format('TCP %d UDP %d ICMP %d bytes',
+ tonumber(tcp or 0), tonumber(udp or 0), tonumber(icmp or 0)))
+ S.sleep(1)
+end

examples/lua/tracepoint-offcputime.lua

@@ -0,0 +1,80 @@
+#!/usr/bin/env bcc-lua
+--[[
+Copyright 2016 Marek Vavrusa <[email protected]>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http:https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]
+-- Summarize off-CPU time by stack trace
+-- Related tool: https://github.com/iovisor/bcc/blob/master/tools/offcputime.py
+local ffi = require('ffi')
+local bpf = require('bpf')
+local S = require('syscall')
+-- Create BPF maps
+-- TODO: made smaller to fit default memory limits
+local key_t = 'struct { char name[16]; int32_t stack_id; }'
+local starts = assert(bpf.map('hash', 128, ffi.typeof('uint32_t'), ffi.typeof('uint64_t')))
+local counts = assert(bpf.map('hash', 128, ffi.typeof(key_t), ffi.typeof('uint64_t')))
+local stack_traces = assert(bpf.map('stack_trace', 16))
+-- Open tracepoint and attach BPF program
+-- The 'arg' parses tracepoint format automatically
+local tp = bpf.tracepoint('sched/sched_switch', function (arg)
+ -- Update previous thread sleep time
+ local pid = arg.prev_pid
+ local now = time()
+ starts[pid] = now
+ -- Calculate current thread's delta time
+ pid = arg.next_pid
+ local from = starts[pid]
+ if not from then
+ return 0
+ end
+ local delta = (now - from) / 1000
+ starts[pid] = nil
+ -- Check if the delta is below 1us
+ if delta < 1 then
+ return
+ end
+ -- Create key for this thread
+ local key = ffi.new(key_t)
+ comm(key.name)
+ key.stack_id = stack_id(stack_traces, BPF.F_FAST_STACK_CMP)
+ -- Update current thread off cpu time with delta
+ local val = counts[key]
+ if not val then
+ counts[key] = 0
+ end
+ xadd(counts[key], delta)
+end, 0, -1)
+-- Helper: load kernel symbols
+ffi.cdef 'unsigned long long strtoull(const char *, char **, int);'
+local ksyms = {}
+for l in io.lines('/proc/kallsyms') do
+ local addr, sym = l:match '(%w+) %w (%S+)'
+ if addr then ksyms[ffi.C.strtoull(addr, nil, 16)] = sym end
+end
+-- User-space part of the program
+while true do
+ for k,v in counts.pairs,counts,nil do
+ local s = ''
+ local traces = stack_traces[k.stack_id]
+ if traces then
+ for i, ip in ipairs(traces) do
+ s = s .. string.format(" %-16p %s", ip, ksyms[ip])
+ end
+ end
+ s = s .. string.format(" %-16s %s", "-", ffi.string(k.name))
+ s = s .. string.format(" %d", tonumber(v))
+ print(s)
+ end
+ S.sleep(1)
+end