Merge from /nmap-exp/luis/nmap-os6.

svn merge --ignore-ancestry svn:https://svn.insecure.org/nmap@26621 svn:https://svn.insecure.org/nmap-exp/luis/nmap-os6

This is the IPv6 OS detection branch. "nmap -6 -O" works now, though at
this point it only prints fingerprints and not OS guesses, because we
need to collect more submissions.

FingerPrintResults.cc

@@ -104,12 +104,18 @@ FingerPrintResults::FingerPrintResults() {
  osscan_opentcpport = osscan_closedtcpport = osscan_closedudpport = -1;
  distance = -1;
  distance_guess = -1;
- FPs = (FingerPrint **) safe_zalloc(o.maxOSTries() * sizeof(FingerPrint *));
  maxTimingRatio = 0;
- numFPs = 0;
 }
 
 FingerPrintResults::~FingerPrintResults() {
+}
+
+FingerPrintResultsIPv4::FingerPrintResultsIPv4() {
+ FPs = (FingerPrint **) safe_zalloc(o.maxOSTries() * sizeof(FingerPrint *));
+ numFPs = 0;
+}
+
+FingerPrintResultsIPv4::~FingerPrintResultsIPv4() {
  int i;
 
  /* Free OS fingerprints of OS scanning was done */
@@ -121,6 +127,25 @@ FingerPrintResults::~FingerPrintResults() {
  free(FPs);
 }
 
+FingerPrintResultsIPv6::FingerPrintResultsIPv6() {
+ unsigned int i;
+
+ begin_time.tv_sec = 0;
+ begin_time.tv_usec = 0;
+ for (i = 0; i < sizeof(fp_responses) / sizeof(*fp_responses); i++)
+ fp_responses[i] = NULL;
+ flow_label = 0;
+}
+
+FingerPrintResultsIPv6::~FingerPrintResultsIPv6() {
+ unsigned int i;
+
+ for (i = 0; i < sizeof(fp_responses) / sizeof(*fp_responses); i++) {
+ if (fp_responses[i])
+ delete fp_responses[i];
+ }
+}
+
 const struct OS_Classification_Results *FingerPrintResults::getOSClassification() {
  if (!isClassified) { populateClassification(); isClassified = true; }
  return &OSR;
@@ -178,6 +203,19 @@ const char *FingerPrintResults::OmitSubmissionFP() {
  return NULL;
 }
 
+/* IPv6 classification is more robust to errors than IPv4, so apply less
+ stringent conditions than the general OmitSubmissionFP. */
+const char *FingerPrintResultsIPv6::OmitSubmissionFP() {
+ static char reason[128];
+
+ if (o.scan_delay > 500) { // This can screw up the sequence timing
+ Snprintf(reason, sizeof(reason), "Scan delay (%d) is greater than 500", o.scan_delay);
+ return reason;
+ }
+
+ return NULL;
+}
+
 
 /* Goes through fingerprinting results to populate OSR */
 void FingerPrintResults::populateClassification() {
@@ -196,8 +234,8 @@ void FingerPrintResults::populateClassification() {
 
  for(printno = 0; printno < num_matches; printno++) {
  // a single print may have multiple classifications
- for (osclass = prints[printno]->OS_class.begin();
- osclass != prints[printno]->OS_class.end();
+ for (osclass = matches[printno]->OS_class.begin();
+ osclass != matches[printno]->OS_class.end();
  osclass++) {
  if (!classAlreadyExistsInResults(&*osclass)) {
  // Then we have to add it ... first ensure we have room
@@ -217,7 +255,7 @@ void FingerPrintResults::populateClassification() {
  // OK, we will add the new class
  OSR.OSC[OSR.OSC_num_matches] = &*osclass;
  OSR.OSC_Accuracy[OSR.OSC_num_matches] = accuracy[printno];
- if (accuracy[printno] == 1.0) OSR.OSC_num_perfect_matches++;
+ if (printno < num_perfect_matches) OSR.OSC_num_perfect_matches++;
  OSR.OSC_num_matches++;
  }
  }

FingerPrintResults.h

@@ -97,6 +97,7 @@ class FingerPrintResults;
 
 #include "nmap.h"
 #include "global_structures.h"
+#include "FPEngine.h"
 
 /* Maximum number of results allowed in one of these things ... */
 #define MAX_FP_RESULTS 36
@@ -112,11 +113,11 @@ struct OS_Classification_Results {
 class FingerPrintResults {
  public: /* For now ... a lot of the data members should be made private */
  FingerPrintResults();
- ~FingerPrintResults();
+ virtual ~FingerPrintResults();
 
  double accuracy[MAX_FP_RESULTS]; /* Percentage of match (1.0 == perfect 
  match) in same order as pritns[] below */
- FingerPrint *prints[MAX_FP_RESULTS]; /* ptrs to matching references -- 
+ FingerMatch *matches[MAX_FP_RESULTS]; /* ptrs to matching references -- 
  highest accuracy matches first */
  int num_perfect_matches; /* Number of 1.0 accuracy matches in prints[] */
  int num_matches; /* Total number of matches in prints[] */
@@ -147,14 +148,13 @@ class FingerPrintResults {
  consistancy standpoint. */
  double maxTimingRatio;
 
- FingerPrint **FPs; /* Fingerprint data obtained from host */
- int numFPs;
-
 /* If the fingerprint is of potentially poor quality, we don't want to
  print it and ask the user to submit it. In that case, the reason
  for skipping the FP is returned as a static string. If the FP is
  great and should be printed, NULL is returned. */
- const char *OmitSubmissionFP();
+ virtual const char *OmitSubmissionFP();
+
+ virtual const char *merge_fpr(const Target *currenths, bool isGoodFP, bool wrapit) const = 0;
 
  private:
  bool isClassified; // Whether populateClassification() has been called
@@ -165,4 +165,27 @@ class FingerPrintResults {
  struct OS_Classification_Results OSR;
 };
 
+class FingerPrintResultsIPv4 : public FingerPrintResults {
+public:
+ FingerPrint **FPs; /* Fingerprint data obtained from host */
+ int numFPs;
+
+ FingerPrintResultsIPv4();
+ virtual ~FingerPrintResultsIPv4();
+ const char *merge_fpr(const Target *currenths, bool isGoodFP, bool wrapit) const;
+};
+
+class FingerPrintResultsIPv6 : public FingerPrintResults {
+public:
+ FPResponse *fp_responses[NUM_FP_PROBES_IPv6];
+ struct timeval begin_time;
+ /* The flow label we set in our sent packets, for calculating offsets later. */
+ unsigned int flow_label;
+
+ FingerPrintResultsIPv6();
+ virtual ~FingerPrintResultsIPv6();
+ const char *OmitSubmissionFP();
+ const char *merge_fpr(const Target *currenths, bool isGoodFP, bool wrapit) const;
+};
+
 #endif /* FINGERPRINTRESULTS_H */

Makefile.in

@@ -17,8 +17,10 @@ NMAPDEVDIR=~/nmap-private-dev
 export NBASEDIR=@NBASEDIR@
 export NSOCKDIR=@NSOCKDIR@
 export LIBLUADIR = @LIBLUADIR@
+export LIBLINEARDIR = @LIBLINEARDIR@
 export NDIR=$(shell pwd)
 export LIBLUA_LIBS = @LIBLUA_LIBS@
+export LIBLINEAR_LIBS = @LIBLINEAR_LIBS@
 export NCATDIR=@NCATDIR@
 CC = @CC@
 CXX = @CXX@
@@ -47,7 +49,7 @@ export CFLAGS = $(CXXFLAGS)
 # CFLAGS = $(DEFS) $(INCLS)
 STATIC =
 LDFLAGS = @LDFLAGS@ $(DBGFLAGS) $(STATIC)
-LIBS = @LIBNBASE_LIBS@ @LIBNSOCK_LIBS@ @LIBPCRE_LIBS@ @LIBPCAP_LIBS@ $(OPENSSL_LIBS) libnetutil/libnetutil.a @LIBDNET_LIBS@ @LIBLUA_LIBS@ @LIBS@
+LIBS = @LIBNBASE_LIBS@ @LIBNSOCK_LIBS@ @LIBPCRE_LIBS@ @LIBPCAP_LIBS@ $(OPENSSL_LIBS) libnetutil/libnetutil.a @LIBDNET_LIBS@ @LIBLUA_LIBS@ @LIBLINEAR_LIBS@ @LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 # LIBS = -lefence @LIBS@
 # LIBS = -lrmalloc @LIBS@
@@ -93,20 +95,20 @@ NSE_OBJS+=nse_openssl.o nse_ssl_cert.o
 endif
 endif
 
-export SRCS = main.cc nmap.cc targets.cc tcpip.cc nmap_error.cc utils.cc idle_scan.cc osscan.cc osscan2.cc output.cc payload.cc scan_engine.cc timing.cc charpool.cc services.cc protocols.cc nmap_rpc.cc portlist.cc NmapOps.cc TargetGroup.cc Target.cc FingerPrintResults.cc service_scan.cc NmapOutputTable.cc MACLookup.cc nmap_tty.cc nmap_dns.cc traceroute.cc portreasons.cc xml.cc $(NSE_SRC) @COMPAT_SRCS@
+export SRCS = main.cc nmap.cc targets.cc tcpip.cc nmap_error.cc utils.cc idle_scan.cc osscan.cc osscan2.cc FPEngine.cc FPmodel.cc output.cc payload.cc scan_engine.cc timing.cc charpool.cc services.cc protocols.cc nmap_rpc.cc portlist.cc NmapOps.cc TargetGroup.cc Target.cc FingerPrintResults.cc service_scan.cc NmapOutputTable.cc MACLookup.cc nmap_tty.cc nmap_dns.cc traceroute.cc portreasons.cc xml.cc $(NSE_SRC) @COMPAT_SRCS@
 
-export HDRS = charpool.h FingerPrintResults.h global_structures.h idle_scan.h MACLookup.h nmap_amigaos.h nmap_dns.h nmap_error.h nmap.h NmapOps.h NmapOutputTable.h nmap_rpc.h nmap_tty.h nmap_winconfig.h osscan.h osscan2.h output.h payload.h portlist.h protocols.h scan_engine.h service_scan.h services.h TargetGroup.h Target.h targets.h tcpip.h timing.h utils.h traceroute.h portreasons.h xml.h $(NSE_HDRS)
+export HDRS = charpool.h FingerPrintResults.h global_structures.h idle_scan.h MACLookup.h nmap_amigaos.h nmap_dns.h nmap_error.h nmap.h NmapOps.h NmapOutputTable.h nmap_rpc.h nmap_tty.h nmap_winconfig.h osscan.h osscan2.h FPEngine.h output.h payload.h portlist.h protocols.h scan_engine.h service_scan.h services.h TargetGroup.h Target.h targets.h tcpip.h timing.h utils.h traceroute.h portreasons.h xml.h $(NSE_HDRS)
 
-OBJS = main.o nmap.o targets.o tcpip.o nmap_error.o utils.o idle_scan.o osscan.o osscan2.o output.o payload.o scan_engine.o timing.o charpool.o services.o protocols.o nmap_rpc.o portlist.o NmapOps.o TargetGroup.o Target.o FingerPrintResults.o service_scan.o NmapOutputTable.o MACLookup.o nmap_tty.o nmap_dns.o traceroute.o portreasons.o xml.o $(NSE_OBJS) @COMPAT_OBJS@
+OBJS = main.o nmap.o targets.o tcpip.o nmap_error.o utils.o idle_scan.o osscan.o osscan2.o FPEngine.o FPmodel.o output.o payload.o scan_engine.o timing.o charpool.o services.o protocols.o nmap_rpc.o portlist.o NmapOps.o TargetGroup.o Target.o FingerPrintResults.o service_scan.o NmapOutputTable.o MACLookup.o nmap_tty.o nmap_dns.o traceroute.o portreasons.o xml.o $(NSE_OBJS) @COMPAT_OBJS@
 
 # %.o : %.cc -- nope this is a GNU extension
 .cc.o:
  $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $< -o $@
 
-all: @LUA_BUILD@ @PCAP_BUILD@ @PCRE_BUILD@ @DNET_BUILD@ @NBASE_BUILD@ @NSOCK_BUILD@ @NCAT_BUILD@ netutil_build
+all: @LUA_BUILD@ @LIBLINEAR_BUILD@ @PCAP_BUILD@ @PCRE_BUILD@ @DNET_BUILD@ @NBASE_BUILD@ @NSOCK_BUILD@ @NCAT_BUILD@ netutil_build
  $(MAKE) $(TARGET) $(BUILDZENMAP) $(BUILDNDIFF) $(BUILDNPING)
 
-$(TARGET): @LUA_DEPENDS@ @PCAP_DEPENDS@ @PCRE_DEPENDS@ @DNET_DEPENDS@ $(NBASEDIR)/libnbase.a $(NSOCKDIR)/src/libnsock.a libnetutil/libnetutil.a $(OBJS)
+$(TARGET): @LUA_DEPENDS@ @LIBLINEAR_DEPENDS@ @PCAP_DEPENDS@ @PCRE_DEPENDS@ @DNET_DEPENDS@ $(NBASEDIR)/libnbase.a $(NSOCKDIR)/src/libnsock.a libnetutil/libnetutil.a $(OBJS)
  @echo Compiling nmap
  rm -f $@
  $(CXX) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
@@ -138,6 +140,9 @@ ncat_build: $(NCATDIR)/Makefile nbase_build nsock_build $(NCATDIR)/ncat.h
 lua_build: $(LIBLUADIR)/Makefile
  @echo Compiling liblua; cd $(LIBLUADIR) && $(MAKE) liblua.a CC="$(CC)" MYCFLAGS="$(CFLAGS) @LUA_CFLAGS@"
 
+liblinear_build: $(LIBLINEARDIR)/Makefile
+ @echo Compiling liblinear; cd $(LIBLINEARDIR) && $(MAKE) liblinear.a CC="$(CC)" CFLAGS="$(CFLAGS)"
+
 #$(LIBPCAPDIR)/Makefile:
 # @echo Configuring libpcap; cd $(LIBPCAPDIR); ./configure
 
@@ -166,7 +171,7 @@ release-rpms:
 web:
  cd $(NMAPDEVDIR) && $(MAKE) web
 
-clean: @LUA_CLEAN@ @PCAP_CLEAN@ @PCRE_CLEAN@ @DNET_CLEAN@ nsock_clean nbase_clean netutil_clean my_clean @NPING_CLEAN@ @ZENMAP_CLEAN@ @NCAT_CLEAN@
+clean: @LUA_CLEAN@ @LIBLINEAR_CLEAN@ @PCAP_CLEAN@ @PCRE_CLEAN@ @DNET_CLEAN@ nsock_clean nbase_clean netutil_clean my_clean @NPING_CLEAN@ @ZENMAP_CLEAN@ @NCAT_CLEAN@
 
 my_clean:
  rm -f dependencies.mk makefile.dep
@@ -187,6 +192,8 @@ ncat_clean:
  -cd $(NCATDIR) && $(MAKE) clean
 lua_clean:
  -cd $(LIBLUADIR) && $(MAKE) clean
+liblinear_clean:
+ -cd $(LIBLINEARDIR) && $(MAKE) clean
 zenmap_clean:
  -cd $(ZENMAPDIR) && $(PYTHON) setup.py clean --all
 ndiff_clean:
@@ -201,6 +208,7 @@ dnet_dist_clean:
  -cd $(LIBDNETDIR) && $(MAKE) distclean
 lua_dist_clean:
  -cd $(LIBLUADIR) && $(MAKE) clean
+liblinear_dist_clean: liblinear_clean
 nbase_dist_clean:
  -cd $(NBASEDIR) && $(MAKE) distclean
 nsock_dist_clean:
@@ -218,7 +226,7 @@ nping_dist_clean:
 debugclean:
  rm -f *.gcov *.gcda *.gcno gmon.out
 
-distclean: my_clean my_distclean @LUA_DIST_CLEAN@ @PCAP_DIST_CLEAN@ @PCRE_DIST_CLEAN@ @DNET_DIST_CLEAN@ @NPING_DIST_CLEAN@ @ZENMAP_DIST_CLEAN@ @NCAT_DIST_CLEAN@ nsock_dist_clean nbase_dist_clean netutil_dist_clean
+distclean: my_clean my_distclean @LUA_DIST_CLEAN@ @LIBLINEAR_DIST_CLEAN@ @PCAP_DIST_CLEAN@ @PCRE_DIST_CLEAN@ @DNET_DIST_CLEAN@ @NPING_DIST_CLEAN@ @ZENMAP_DIST_CLEAN@ @NCAT_DIST_CLEAN@ nsock_dist_clean nbase_dist_clean netutil_dist_clean
 
 my_distclean:
  rm -f Makefile Makefile.bak makefile.dep nmap_config.h stamp-h stamp-h.in \

NmapOps.cc

@@ -326,6 +326,7 @@ void NmapOps::Initialize() {
  ipopt_firsthop = 0;
  ipopt_lasthop = 0; 
  release_memory = false;
+ disable_ipv6_os_results = true;
  topportlevel = -1;
 #ifndef NOLUA
  script = 0;
@@ -526,7 +527,7 @@ dialog where you can start NPF if you have administrator privileges.";
  fatal("--min-rate=%g must be less than or equal to --max-rate=%g", min_packet_send_rate, max_packet_send_rate);
  }
 
- if (af() == AF_INET6 && (generate_random_ips|numdecoys|osscan|bouncescan|fragscan|idlescan)) {
+ if (af() == AF_INET6 && (generate_random_ips|numdecoys|bouncescan|fragscan|idlescan)) {
  fatal("Sorry -- IPv6 support is currently only available for TCP, UDP, and SCTP port scans and list scan (-sL). OS detection, random targets and decoys are also not supported with IPv6. Further support is under consideration.");
  }
 

NmapOps.h

@@ -187,6 +187,7 @@ class NmapOps {
  int nogcc; /* Turn off group congestion control with --nogcc */
  int generate_random_ips; /* -iR option */
  FingerPrintDB *reference_FPs; /* Used in the new OS scan system. */
+ std::vector<FingerMatch> os_labels_ipv6;
  u16 magic_port;
  unsigned short magic_port_set; /* Was this set by user? */
 
@@ -351,6 +352,11 @@ class NmapOps {
  bool noninteractive;
 
  bool release_memory; /* suggest to release memory before quitting. used to find memory leaks. */
+
+ /* This is a temporary option that disables the printing of IPv6 OS results,
+ to allow collection of more fingerprints. */
+ bool disable_ipv6_os_results;
+
  private:
  int max_os_tries;
  int max_rtt_timeout;

Target.cc

@@ -113,6 +113,7 @@ void Target::Initialize() {
  targetname = NULL;
  memset(&seq, 0, sizeof(seq));
  distance = -1;
+ distance_calculation_method = DIST_METHOD_NONE;
  FPR = NULL;
  osscan_flag = OS_NOTPERF;
  weird_responses = flags = 0;
@@ -124,6 +125,7 @@ void Target::Initialize() {
  targetsocklen = sourcesocklen = nexthopsocklen = 0;
  directly_connected = -1;
  targetipstring[0] = '\0';
+ sourceipstring[0] = '\0';
  nameIPBuf = NULL;
  memset(&MACaddress, 0, sizeof(MACaddress));
  memset(&SrcMACaddress, 0, sizeof(SrcMACaddress));
@@ -177,7 +179,7 @@ void Target::FreeInternal() {
 
 /* Creates a "presentation" formatted string out of the IPv4/IPv6 address.
  Called when the IP changes */
-void Target::GenerateIPString() {
+void Target::GenerateTargetIPString() {
  struct sockaddr_in *sin = (struct sockaddr_in *) &targetsock;
 #if HAVE_IPV6
  struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) &targetsock;
@@ -195,6 +197,26 @@ void Target::GenerateIPString() {
  }
 }
 
+/* Creates a "presentation" formatted string out of the IPv4/IPv6 address.
+ Called when the IP changes */
+void Target::GenerateSourceIPString() {
+ struct sockaddr_in *sin = (struct sockaddr_in *) &sourcesock;
+#if HAVE_IPV6
+ struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) &sourcesock;
+#endif
+
+ if (inet_ntop(sin->sin_family, (sin->sin_family == AF_INET)?
+ (char *) &sin->sin_addr :
+#if HAVE_IPV6
+ (char *) &sin6->sin6_addr,
+#else
+ (char *) NULL,
+#endif
+ sourceipstring, sizeof(sourceipstring)) == NULL) {
+ fatal("Failed to convert source address to presentation format!?! Error: %s", strerror(socket_errno()));
+ }
+}
+
 /* Returns the address family of the destination address. */
 int Target::af() const {
  return targetsock.ss_family;
@@ -233,7 +255,7 @@ void Target::setTargetSockAddr(const struct sockaddr_storage *ss, size_t ss_len)
  }
  memcpy(&targetsock, ss, ss_len);
  targetsocklen = ss_len;
- GenerateIPString();
+ GenerateTargetIPString();
  /* The ports array needs to know a name too */
  ports.setIdStr(targetipstr());
 }
@@ -286,6 +308,7 @@ void Target::setSourceSockAddr(const struct sockaddr_storage *ss, size_t ss_len)
  assert(ss_len > 0 && ss_len <= sizeof(*ss));
  memcpy(&sourcesock, ss, ss_len);
  sourcesocklen = ss_len;
+ GenerateSourceIPString();
 }
 
 // Returns IPv4 host address or {0} if unavailable.

Target.h

@@ -185,6 +185,8 @@ class Target {
  const struct in6_addr *v6sourceip() const;
  /* The IPv4 or IPv6 literal string for the target host */
  const char *targetipstr() const { return targetipstring; }
+ /* The IPv4 or IPv6 literal string for the source address */
+ const char *sourceipstr() const { return sourceipstring; }
  /* Give the name from the last setHostName() call, which should be
  the name obtained from reverse-resolution (PTR query) of the IP (v4
  or v6). If the name has not been set, or was set to NULL, an empty
@@ -315,12 +317,15 @@ class Target {
  private:
  void Initialize();
  void FreeInternal(); // Free memory allocated inside this object
- // Creates a "presentation" formatted string out of the IPv4/IPv6 address
- void GenerateIPString();
+ // Creates a "presentation" formatted string out of the target's IPv4/IPv6 address
+ void GenerateTargetIPString();
+ // Creates a "presentation" formatted string out of the source IPv4/IPv6 address.
+ void GenerateSourceIPString();
  struct sockaddr_storage targetsock, sourcesock, nexthopsock;
  size_t targetsocklen, sourcesocklen, nexthopsocklen;
  int directly_connected; // -1 = unset; 0 = no; 1 = yes
  char targetipstring[INET6_ADDRSTRLEN];
+ char sourceipstring[INET6_ADDRSTRLEN];
  char *nameIPBuf; /* for the NameIP(void) function to return */
  u8 MACaddress[6], SrcMACaddress[6], NextHopMACaddress[6]; 
  bool MACaddress_set, SrcMACaddress_set, NextHopMACaddress_set;