librdmacm/cmtime: Add OOB sync mechanism for coordination

Add a simple socket connection to sync the client and server
during testing.

When scaling up tests, CM issues are exposed in ways that are
unexpected.  :)  Yes, the problems are being exposed, but
earlier and in more difficult ways to analyze.

Example: A client finishes connecting by sending an RTU
message.  It then immediately starts disconnecting.  This sends
DREQs.  The problem is that the number of messages seen
by the server is now twice as many as the number of
connections.  The result is that DREQs get dropped, which
results in retries, and the client and server move out of sync.
So, although the test shows scaling for connecting, it starts
to break when disconnecting.

The actual synchronization during the test is added as a separate
change.  This patch just lays the foundation.

Signed-off-by: Sean Hefty <[email protected]>

librdmacm/examples/cmtime.c

@@ -50,6 +50,7 @@
 
 static struct rdma_addrinfo hints, *rai;
 static struct rdma_event_channel *channel;
+static int oob_sock = -1;
 static const char *port = "7471";
 static char *dst_addr;
 static char *src_addr;
@@ -828,10 +829,14 @@ static int run_client(int iter)
 {
  int ret;
 
+ ret = oob_client_setup(dst_addr, port, &oob_sock);
+ if (ret)
+ return ret;
+
  printf("Client warmup\n");
  ret = client_connect(1);
  if (ret)
- return ret;
+ goto out;
 
  if (!mimic) {
  printf("Connect (%d) QPs test\n", iter);
@@ -842,7 +847,7 @@ static int run_client(int iter)
  }
  ret = client_connect(iter);
  if (ret)
- return ret;
+ goto out;
 
  show_perf();
 
@@ -851,9 +856,11 @@ static int run_client(int iter)
  mimic_qp_delay = 0;
  ret = client_connect(iter);
  if (ret)
- return ret;
+ goto out;
 
  show_perf();
+out:
+ close(oob_sock);
  return 0;
 }
 
@@ -862,10 +869,15 @@ static int run_server(int iter)
  struct rdma_cm_id *listen_id;
  int ret;
 
+ /* Make sure we're ready for RDMA prior to any OOB sync */
  ret = server_listen(&listen_id);
  if (ret)
  return ret;
 
+ ret = oob_server_setup(src_addr, port, &oob_sock);
+ if (ret)
+ goto out;
+
  printf("Server warmup\n");
  ret = server_connect(1);
  if (ret)
@@ -893,6 +905,7 @@ static int run_server(int iter)
 
  show_perf();
 out:
+ close(oob_sock);
  rdma_destroy_id(listen_id);
  return ret;
 }

librdmacm/examples/common.c

@@ -38,7 +38,10 @@
 #include <errno.h>
 #include <sys/types.h>
 #include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <netdb.h>
+#include <unistd.h>
 
 #include <rdma/rdma_cma.h>
 #include "common.h"
@@ -180,3 +183,114 @@ struct rdma_event_channel *create_event_channel(void)
  }
  return channel;
 }
+
+int oob_server_setup(const char *src_addr, const char *port, int *sock)
+{
+ struct addrinfo hint = {}, *ai;
+ int listen_sock;
+ int optval = 1;
+ int ret;
+
+ hint.ai_flags = AI_PASSIVE;
+ hint.ai_family = AF_INET;
+ hint.ai_socktype = SOCK_STREAM;
+ ret = getaddrinfo(src_addr, port, &hint, &ai);
+ if (ret) {
+ printf("getaddrinfo error: %s\n", gai_strerror(ret));
+ return ret;
+ }
+
+ listen_sock = socket(ai->ai_family, ai->ai_socktype, 0);
+ if (listen_sock == -1) {
+ ret = -errno;
+ goto free;
+ }
+
+ setsockopt(listen_sock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
+ ret = bind(listen_sock, ai->ai_addr, ai->ai_addrlen);
+ if (ret) {
+ ret = -errno;
+ goto close;
+ }
+
+ ret = listen(listen_sock, 1);
+ if (ret) {
+ ret = -errno;
+ goto close;
+ }
+
+ *sock = accept(listen_sock, NULL, NULL);
+ if (*sock == -1)
+ ret = -errno;
+ setsockopt(*sock, IPPROTO_TCP, TCP_NODELAY, &optval, sizeof(optval));
+
+close:
+ close(listen_sock);
+free:
+ freeaddrinfo(ai);
+ return ret;
+}
+
+int oob_client_setup(const char *dst_addr, const char *port, int *sock)
+{
+ struct addrinfo hint = {}, *ai;
+ int nodelay = 1;
+ int ret;
+
+ hint.ai_family = AF_INET;
+ hint.ai_socktype = SOCK_STREAM;
+ ret = getaddrinfo(dst_addr, port, &hint, &ai);
+ if (ret) {
+ printf("getaddrinfo error: %s\n", gai_strerror(ret));
+ return ret;
+ }
+
+ *sock = socket(ai->ai_family, ai->ai_socktype, 0);
+ if (*sock == -1) {
+ ret = -errno;
+ goto out;
+ }
+ setsockopt(*sock, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof(nodelay));
+
+ ret = connect(*sock, ai->ai_addr, ai->ai_addrlen);
+out:
+ freeaddrinfo(ai);
+ return ret;
+}
+
+int oob_sendrecv(int sock, char val)
+{
+ char c = val;
+ ssize_t ret;
+
+ ret = send(sock, (void *) &c, sizeof(c), 0);
+ if (ret != sizeof(c))
+ return -errno;
+
+ ret = recv(sock, (void *) &c, sizeof(c), 0);
+ if (ret != sizeof(c))
+ return -errno;
+
+ if (c != val)
+ return -EINVAL;
+ return 0;
+}
+
+int oob_recvsend(int sock, char val)
+{
+ char c = 0;
+ ssize_t ret;
+
+ ret = recv(sock, (void *) &c, sizeof(c), 0);
+ if (ret != sizeof(c))
+ return -errno;
+
+ if (c != val)
+ return -EINVAL;
+
+ ret = send(sock, (void *) &c, sizeof(c), 0);
+ if (ret != sizeof(c))
+ return -errno;
+
+ return 0;
+}

librdmacm/examples/common.h

@@ -98,6 +98,11 @@ enum rs_optimization {
 int get_rdma_addr(const char *src, const char *dst, const char *port,
  struct rdma_addrinfo *hints, struct rdma_addrinfo **rai);
 
+int oob_server_setup(const char *src_addr, const char *port, int *sock);
+int oob_client_setup(const char *dst_addr, const char *port, int *sock);
+int oob_sendrecv(int sock, char val);
+int oob_recvsend(int sock, char val);
+
 void size_str(char *str, size_t ssize, long long size);
 void cnt_str(char *str, size_t ssize, long long cnt);
 int size_to_count(int size);
@@ -130,4 +135,4 @@ static inline int sleep_us(unsigned int time_us)
  spec.tv_sec = 0;
  spec.tv_nsec = time_us * 1000;
  return nanosleep(&spec, NULL);
-}
+}