New C backend for datafiles.lua; avoid copies of large data sets

Makefile.in

@@ -88,9 +88,9 @@ UNINSTALLNDIFF=@UNINSTALLNDIFF@
 UNINSTALLNPING=@UNINSTALLNPING@
 
 ifneq (@NOLUA@,yes)
-NSE_SRC=nse_main.cc nse_utility.cc nse_nsock.cc nse_dnet.cc nse_fs.cc nse_nmaplib.cc nse_debug.cc nse_pcrelib.cc nse_lpeg.cc
-NSE_HDRS=nse_main.h nse_utility.h nse_nsock.h nse_dnet.h nse_fs.h nse_nmaplib.h nse_debug.h nse_pcrelib.h nse_lpeg.h
-NSE_OBJS=nse_main.o nse_utility.o nse_nsock.o nse_dnet.o nse_fs.o nse_nmaplib.o nse_debug.o nse_pcrelib.o nse_lpeg.o
+NSE_SRC=nse_main.cc nse_utility.cc nse_nsock.cc nse_db.cc nse_dnet.cc nse_fs.cc nse_nmaplib.cc nse_debug.cc nse_pcrelib.cc nse_lpeg.cc
+NSE_HDRS=nse_main.h nse_utility.h nse_nsock.h nse_db.h nse_dnet.h nse_fs.h nse_nmaplib.h nse_debug.h nse_pcrelib.h nse_lpeg.h
+NSE_OBJS=nse_main.o nse_utility.o nse_nsock.o nse_db.o nse_dnet.o nse_fs.o nse_nmaplib.o nse_debug.o nse_pcrelib.o nse_lpeg.o
 ifneq (@OPENSSL_LIBS@,)
 NSE_SRC+=nse_openssl.cc nse_ssl_cert.cc
 NSE_HDRS+=nse_openssl.h nse_ssl_cert.h

mswin32/nmap.vcxproj

@@ -204,6 +204,7 @@
  <ClCompile Include="..\NmapOps.cc" />
  <ClCompile Include="..\NmapOutputTable.cc" />
  <ClCompile Include="..\nse_debug.cc" />
+ <ClCompile Include="..\nse_db.cc" />
  <ClCompile Include="..\nse_fs.cc" />
  <ClCompile Include="..\nse_libssh2.cc" />
  <ClCompile Include="..\nse_lpeg.cc" />
@@ -263,6 +264,7 @@
  <ClInclude Include="..\NmapOps.h" />
  <ClInclude Include="..\NmapOutputTable.h" />
  <ClInclude Include="..\nse_debug.h" />
+ <ClInclude Include="..\nse_db.h" />
  <ClInclude Include="..\nse_fs.h" />
  <ClInclude Include="..\nse_libssh2.h" />
  <ClInclude Include="..\nse_lpeg.h" />

nse_db.cc

@@ -0,0 +1,113 @@
+#include <nbase.h>
+
+#include "nse_lua.h"
+#include "MACLookup.h"
+#include "services.h"
+#include "protocols.h"
+
+static inline u8 nibble(char hex) {
+ return (hex & 0xf) + ((hex & 0x40) ? 9 : 0);
+}
+
+static int l_mac2corp (lua_State *L)
+{
+ size_t len = 0;
+ u8 prefix[6] = {0}; // allow a whole MAC addr.
+ size_t i = 0;
+ size_t j = 0;
+ const char *buf = luaL_checklstring(L, 1, &len);
+
+ if (len == 6) {
+ // Option 1: 6-byte raw MAC
+ lua_pushstring(L, MACPrefix2Corp((u8 *)buf));
+ return 1;
+ }
+
+ // Try for hex string.
+ for (i = 0; i + 1 < len && j < 6; i+=2 ) {
+ if (buf[i] == ':' && i + 2 < len) {
+ i++;
+ }
+ if (isxdigit(buf[i]) && isxdigit(buf[i+1])) {
+ prefix[j++] = (nibble(buf[i]) << 4) + nibble(buf[i+1]);
+ }
+ else {
+ break;
+ }
+ }
+ // Require exactly 6 bytes result and used the whole input
+ if (j == 6 && i >= len) {
+ lua_pushstring(L, MACPrefix2Corp(prefix));
+ return 1;
+ }
+ return luaL_error(L, "Expected a 6-byte MAC address");
+}
+
+static int l_getservbyport (lua_State *L)
+{
+ const struct nservent *serv = NULL;
+ static const u16 proto[] = {IPPROTO_TCP, IPPROTO_UDP, IPPROTO_SCTP};
+ static const char * op[] = {"tcp", "udp", "sctp"};
+ lua_Integer port = luaL_checkinteger(L, 1);
+ int i = luaL_checkoption(L, 2, NULL, op);
+
+ if (port < 0 || port > 0xffff) {
+ return luaL_error(L, "Port number out of range");
+ }
+
+ serv = nmap_getservbyport((u16) port, proto[i]);
+ if (serv == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushstring(L, serv->s_name);
+ }
+ return 1;
+}
+
+static int l_getprotbynum (lua_State *L)
+{
+ const struct nprotoent *proto = NULL;
+ lua_Integer num = luaL_checkinteger(L, 1);
+
+ if (num < 0 || num > 0xff) {
+ return luaL_error(L, "Protocol number out of range");
+ }
+
+ proto = nmap_getprotbynum(num);
+ if (proto == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushstring(L, proto->p_name);
+ }
+ return 1;
+}
+
+static int l_getprotbyname (lua_State *L)
+{
+ const struct nprotoent *proto = NULL;
+ const char *name = luaL_checkstring(L, 1);
+
+ proto = nmap_getprotbyname(name);
+ if (proto == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushinteger(L, proto->p_proto);
+ }
+ return 1;
+}
+
+int luaopen_db (lua_State *L)
+{
+ static const luaL_Reg dblib [] = {
+ {"mac2corp", l_mac2corp},
+ {"getservbyport", l_getservbyport},
+ {"getprotbynum", l_getprotbynum},
+ {"getprotbyname", l_getprotbyname},
+ {NULL, NULL}
+ };
+ luaL_newlib(L, dblib);
+ return 1;
+}

nse_db.h

@@ -0,0 +1,7 @@
+#ifndef NSE_DB
+#define NSE_DB
+
+#define NSE_DBLIBNAME "nmapdb"
+LUALIB_API int luaopen_db (lua_State *L);
+
+#endif

nse_main.cc

@@ -11,6 +11,7 @@
 
 #include "nse_main.h"
 #include "nse_utility.h"
+#include "nse_db.h"
 #include "nse_fs.h"
 #include "nse_nsock.h"
 #include "nse_nmaplib.h"
@@ -559,6 +560,7 @@ static void set_nmap_libraries (lua_State *L)
  static const luaL_Reg libs[] = {
  {NSE_PCRELIBNAME, luaopen_pcrelib},
  {NSE_NMAPLIBNAME, luaopen_nmap},
+ {NSE_DBLIBNAME, luaopen_db},
  {LFSLIBNAME, luaopen_lfs},
  {LPEGLIBNAME, luaopen_lpeg},
 #ifdef HAVE_LIBSSH2

nselib/datafiles.lua

@@ -15,6 +15,8 @@ local nmap = require "nmap"
 local stdnse = require "stdnse"
 local string = require "string"
 local table = require "table"
+-- mostly undocumented library for direct lookups in Nmap datafiles:
+local nmapdb = require "nmapdb"
 _ENV = stdnse.module("datafiles", stdnse.seeall)
 
 
@@ -72,59 +74,71 @@ parse_rpc = function()
  return parse_and_cache("nmap-rpc")
 end
 
+local prohibited = function()
+ error("Invalid function")
+end
+local services_table = {}
+local portlookup_mt = {
+ __index = function(t, port)
+ return nmapdb.getservbyport(port, rawget(t, "proto"))
+ end,
+ __newindex = prohibited,
+}
+for _, proto in ipairs({"tcp", "udp", "sctp"}) do
+ services_table[proto] = setmetatable({proto=proto}, portlookup_mt)
+end
 
 ---
 -- Read and parse <code>nmap-services</code>.
 --
--- On success, return true and a table containing two subtables, indexed by the
--- keys "tcp" and "udp". The <code>tcp</code> subtable maps TCP port numbers to
--- service names, and the <code>udp</code> subtable is the same for UDP. You can
--- pass "tcp" or "udp" as an argument to <code>parse_services</code> to get
+-- On success, return true and a table containing subtables indexed by the
+-- keys "tcp", "udp", and "sctp". You can
+-- pass a protocol name as an argument to <code>parse_services</code> to get
 -- only one of the results tables.
--- @param protocol The protocol table to return (<code>"tcp"</code> or
+-- @param protocol Optional: The protocol table to return (e.g. <code>"tcp"</code> or
 -- <code>"udp"</code>).
 -- @return Status (true or false).
 -- @return Table (if status is true) or error string (if status is false).
 -- @see parse_file
 parse_services = function(protocol)
- if protocol and protocol ~= "tcp" and protocol ~= "udp" then
- return false, "Bad protocol for nmap-services: use tcp or udp"
- end
-
- local services_table
- nmap.registry.datafiles = nmap.registry.datafiles or {}
- nmap.registry.datafiles.services = nmap.registry.datafiles.services or {}
+ local t
  if protocol then
- if not nmap.registry.datafiles.services[protocol] then
- local status
- status, nmap.registry.datafiles.services[protocol] = parse_file("nmap-services", protocol)
- if not status then
- return false, "Error parsing nmap-services"
- end
+ t = services_table[protocol]
+ if not t then
+ return false, "Bad protocol for nmap-services"
  end
- services_table = nmap.registry.datafiles.services[protocol]
  else
- local status
- status, nmap.registry.datafiles.services = parse_file("nmap-services")
- if not status then
- return false, "Error parsing nmap-services"
- end
- services_table = nmap.registry.datafiles.services
+ t = services_table
  end
 
- return true, services_table
+ return true, t
 end
 
 
+local mac_table = setmetatable({}, {
+ __index = function(t, mac)
+ if #mac < 6 then
+ -- probably binary
+ mac = mac .. ("\0"):rep(6 - #mac)
+ elseif #mac < 12 then
+ -- probably hex
+ mac = mac .. ("0"):rep(12 - #mac)
+ end
+ return nmapdb.mac2corp(mac)
+ end,
+ __newindex = prohibited,
+})
 ---
 -- Read and parse <code>nmap-mac-prefixes</code>.
 --
--- On success, return true and a table mapping 3 byte MAC prefixes to manufacturer names.
+-- On success, return true and a table mapping MAC prefixes to manufacturer
+-- names. The whole MAC can also be used as a key, since the table calls an
+-- internal Nmap function to do the lookup.
 -- @return Status (true or false).
 -- @return Table (if status is true) or error string (if status is false).
 -- @see parse_file
 parse_mac_prefixes = function()
- return parse_and_cache("nmap-mac-prefixes")
+ return true, mac_table
 end