Merge "/bin/wifi:  log hostapd/wpa_supplicant startup times."
diff --git a/cmds/Makefile b/cmds/Makefile
index 4bcca65..aabebe5 100644
--- a/cmds/Makefile
+++ b/cmds/Makefile
@@ -184,11 +184,11 @@
 	echo "Building .pb.cc"
 	$(HOST_PROTOC) --cpp_out=. $<
 
-host-isoping isoping: LIBS+=$(RT) -lm -lstdc++
+host-isoping isoping: LIBS+=$(RT) -lm -lstdc++ -lcrypto -lpthread
 host-isoping: host-isoping.o host-isoping_main.o
 host-isoping_test.o: CXXFLAGS += -D WVTEST_CONFIGURED -I ../wvtest/cpp
 host-isoping_test.o: isoping.cc
-host-isoping_test: LIBS+=$(HOST_LIBS) -lm -lstdc++
+host-isoping_test: LIBS+=$(HOST_LIBS) -lm -lstdc++ -lcrypto
 host-isoping_test: host-isoping_test.o host-isoping.o host-wvtestmain.o host-wvtest.o
 host-isostream isostream: LIBS+=$(RT)
 host-diskbench diskbench: LIBS+=-lpthread $(RT)
diff --git a/cmds/isoping.cc b/cmds/isoping.cc
index c6b123e..d7b9d1c 100644
--- a/cmds/isoping.cc
+++ b/cmds/isoping.cc
@@ -27,6 +27,7 @@
 #include "isoping.h"
 
 #include <arpa/inet.h>
+#include <assert.h>
 #include <errno.h>
 #include <math.h>
 #include <memory.h>
@@ -65,9 +66,13 @@
 #define DIV(x, y) ((y) ? ((double)(x)/(y)) : 0)
 #define _STR(n) #n
 #define STR(n) _STR(n)
+#ifdef DEBUG
+#define DLOG(args...) fprintf(stderr, args)
+#else
+#define DLOG(args...)
+#endif
 
 // Global flag values.
-int is_server = 1;
 int quiet = 0;
 int ttl = DEFAULT_TTL;
 int want_timestamps = 0;
@@ -81,9 +86,47 @@
   want_to_die = 1;
 }
 
-Session::Session(uint32_t now)
-    : usec_per_pkt(1e6 / packets_per_sec),
+// Render the given sockaddr as a string.  (Uses a static internal buffer
+// which is overwritten each time.)
+static const char *sockaddr_to_str(struct sockaddr *sa) {
+  static char addrbuf[128];
+  void *aptr;
+
+  switch (sa->sa_family) {
+  case AF_INET:
+    aptr = &((struct sockaddr_in *)sa)->sin_addr;
+    break;
+  case AF_INET6:
+    aptr = &((struct sockaddr_in6 *)sa)->sin6_addr;
+    break;
+  default:
+    return "unknown";
+  }
+
+  if (!inet_ntop(sa->sa_family, aptr, addrbuf, sizeof(addrbuf))) {
+    perror("inet_ntop");
+    exit(98);
+  }
+  return addrbuf;
+}
+
+static void debug_print_hex(unsigned char *data, size_t data_len) {
+  for (size_t i = 0; i < data_len; i++) {
+    DLOG("%02x", data[i]);
+    if (i % 8 == 7) {
+      DLOG(" ");
+    }
+  }
+  DLOG("\n");
+}
+
+Session::Session(uint32_t first_send, uint32_t usec_per_pkt,
+                 const struct sockaddr_storage &raddr, size_t raddr_len)
+    : usec_per_pkt(usec_per_pkt),
       usec_per_print(prints_per_sec > 0 ? 1e6 / prints_per_sec : 0),
+      remoteaddr_len(raddr_len),
+      handshake_state(NEW_SESSION),
+      handshake_retry_count(0),
       next_tx_id(1),
       next_rx_id(0),
       next_rxack_id(0),
@@ -92,16 +135,125 @@
       last_rxtime(0),
       min_cycle_rxdiff(0),
       next_cycle(0),
-      next_send(now + usec_per_pkt),
+      next_send(first_send),
       num_lost(0),
       next_txack_index(0),
-      last_print(now - usec_per_pkt),
+      last_print(first_send - usec_per_pkt),
       lat_tx(0), lat_tx_min(0x7fffffff), lat_tx_max(0),
       lat_tx_count(0), lat_tx_sum(0), lat_tx_var_sum(0),
       lat_rx(0), lat_rx_min(0x7fffffff), lat_rx_max(0),
       lat_rx_count(0), lat_rx_sum(0), lat_rx_var_sum(0) {
+  memcpy(&remoteaddr, &raddr, raddr_len);
   memset(&tx, 0, sizeof(tx));
   strcpy(last_ackinfo, "");
+  DLOG("Handshake state: NEW_SESSION\n");
+}
+
+SessionMap::iterator Sessions::NewSession(uint32_t first_send,
+                                          uint32_t usec_per_pkt,
+                                          struct sockaddr_storage *addr,
+                                          socklen_t addr_len) {
+  std::pair<SessionMap::iterator, bool> p = session_map.insert(std::make_pair(
+      *addr, Session(first_send, usec_per_pkt, *addr, addr_len)));
+  next_sends.push(p.first);
+  return p.first;
+}
+
+bool Sessions::CalculateCookie(Packet *p, struct sockaddr_storage *remoteaddr,
+                               size_t remoteaddr_len) {
+  return CalculateCookieWithSecret(p, remoteaddr, remoteaddr_len,
+                                   cookie_secret, sizeof(cookie_secret));
+}
+
+bool Sessions::CalculateCookieWithSecret(Packet *p,
+                                         struct sockaddr_storage *remoteaddr,
+                                         size_t remoteaddr_len,
+                                         unsigned char *secret,
+                                         size_t secret_len) {
+  if (p->packet_type != PACKET_TYPE_HANDSHAKE) {
+    fprintf(stderr, "Tried to create cookie for a non-handshake packet\n");
+    return false;
+  }
+  if (!EVP_DigestInit_ex(&digest_context, md, NULL)) {
+    fprintf(stderr, "Unable to initialize hash digest\n");
+    return false;
+  }
+
+  // Hash the data
+  EVP_DigestUpdate(&digest_context, secret, secret_len);
+  EVP_DigestUpdate(&digest_context, &p->usec_per_pkt, sizeof(p->usec_per_pkt));
+  EVP_DigestUpdate(&digest_context, remoteaddr, remoteaddr_len);
+
+  unsigned int digest_size = 0;
+  EVP_DigestFinal_ex(&digest_context, p->data.handshake.cookie, &digest_size);
+  if (digest_size != COOKIE_SIZE) {
+    fprintf(stderr, "Invalid digest size %d for cookie; expected %d\n",
+            digest_size, COOKIE_SIZE);
+    return false;
+  }
+  p->data.handshake.cookie_epoch = cookie_epoch;
+  return true;
+}
+
+bool Sessions::ValidateCookie(Packet *p, struct sockaddr_storage *addr,
+                              socklen_t addr_len) {
+  if (p->data.handshake.cookie_epoch != cookie_epoch &&
+      p->data.handshake.cookie_epoch != prev_cookie_epoch) {
+    fprintf(stderr, "Obsolete cookie epoch: %d\n",
+            p->data.handshake.cookie_epoch);
+    return false;
+  }
+  Packet golden;
+  golden.packet_type = PACKET_TYPE_HANDSHAKE;
+  golden.usec_per_pkt = p->usec_per_pkt;
+  if (p->data.handshake.cookie_epoch == cookie_epoch) {
+    CalculateCookieWithSecret(&golden, addr, addr_len, cookie_secret,
+                              sizeof(cookie_secret));
+  } else {
+    CalculateCookieWithSecret(&golden, addr, addr_len, prev_cookie_secret,
+                              sizeof(prev_cookie_secret));
+  }
+  DLOG("Handshake: cookie epoch=%d, cookie=0x",
+       p->data.handshake.cookie_epoch);
+  debug_print_hex(p->data.handshake.cookie, sizeof(p->data.handshake.cookie));
+  DLOG("Expected handshake: cookie epoch=%d, cookie=0x",
+       golden.data.handshake.cookie_epoch);
+  debug_print_hex(golden.data.handshake.cookie,
+                  sizeof(golden.data.handshake.cookie));
+  if (memcmp(golden.data.handshake.cookie, p->data.handshake.cookie,
+             COOKIE_SIZE)) {
+    fprintf(stderr, "Invalid cookie in handshake packet from %s\n",
+            sockaddr_to_str((struct sockaddr *)addr));
+    return false;
+  }
+  return true;
+}
+
+void Sessions::MaybeRotateCookieSecrets() {
+  // Round off the unix timestamp to 64 seconds as an epoch, so we don't have to
+  // track which ones we've already used.
+  uint32_t new_epoch = time(NULL) >> 6;
+  if (new_epoch != cookie_epoch) {
+    RotateCookieSecrets(new_epoch);
+  }
+}
+
+void Sessions::RotateCookieSecrets(uint32_t new_epoch) {
+  prev_cookie_epoch = cookie_epoch;
+  memcpy(&prev_cookie_secret[0], &cookie_secret[0],
+         sizeof(prev_cookie_secret));
+  cookie_epoch = new_epoch;
+  NewRandomCookieSecret();
+}
+
+void Sessions::NewRandomCookieSecret() {
+  uint64_t random;
+  for (size_t i = 0; i < sizeof(cookie_secret); i += sizeof(random)) {
+    random = rng();
+    memcpy(&cookie_secret[i], &random,
+           std::min(sizeof(random), sizeof(cookie_secret) - i));
+  }
+  DLOG("Generated new cookie secret.\n");
 }
 
 // Returns the kernel monotonic timestamp in microseconds, truncated to
@@ -163,30 +315,34 @@
 }
 
 
-// Render the given sockaddr as a string.  (Uses a static internal buffer
-// which is overwritten each time.)
-static const char *sockaddr_to_str(struct sockaddr *sa) {
-  static char addrbuf[128];
-  void *aptr;
-
-  switch (sa->sa_family) {
-  case AF_INET:
-    aptr = &((struct sockaddr_in *)sa)->sin_addr;
-    break;
-  case AF_INET6:
-    aptr = &((struct sockaddr_in6 *)sa)->sin6_addr;
-    break;
-  default:
-    return "unknown";
+bool CompareSockaddr::operator()(const struct sockaddr_storage &lhs,
+                                 const struct sockaddr_storage &rhs) {
+  if (lhs.ss_family != rhs.ss_family) {
+    return lhs.ss_family < rhs.ss_family;
   }
-
-  if (!inet_ntop(sa->sa_family, aptr, addrbuf, sizeof(addrbuf))) {
-    perror("inet_ntop");
-    exit(98);
+  if (lhs.ss_family == AF_INET) {
+    const struct sockaddr_in &lhs4 = *(const struct sockaddr_in*)&lhs;
+    const struct sockaddr_in &rhs4 = *(const struct sockaddr_in*)&rhs;
+    long long c = (ntohl(lhs4.sin_addr.s_addr) - ntohl(rhs4.sin_addr.s_addr));
+    if (c == 0) {
+      return ntohs(lhs4.sin_port) < ntohs(rhs4.sin_port);
+    }
+    return c < 0;
+  } else {
+    const struct sockaddr_in6 &lhs6 = *(const struct sockaddr_in6*)&lhs;
+    const struct sockaddr_in6 &rhs6 = *(const struct sockaddr_in6*)&rhs;
+    int c = memcmp(&lhs6.sin6_addr, &rhs6.sin6_addr, sizeof(struct in6_addr));
+    if (c == 0) {
+      return ntohs(lhs6.sin6_port) < ntohs(rhs6.sin6_port);
+    }
+    return c < 0;
   }
-  return addrbuf;
 }
 
+bool CompareNextSend::operator()(const SessionMap::iterator &lhs,
+                                 const SessionMap::iterator &rhs) {
+  return lhs->second.next_send > rhs->second.next_send;
+}
 
 // Print the timestamp corresponding to the current time.
 // Deliberately the same format as tcpdump uses, so we can easily sort and
@@ -213,6 +369,26 @@
   return sqrt(DIV(numer, denom));
 }
 
+static void debug_print_packet(Packet *p) {
+  DLOG("Packet contents: magic=0x%x id=%d usec_per_pkt=%d txtime=%u "
+       "clockdiff=%d num_lost=%d first_ack=%d type=%d\n",
+       ntohl(p->magic), ntohl(p->id), ntohl(p->usec_per_pkt),
+       ntohl(p->txtime), ntohl(p->clockdiff), ntohl(p->num_lost),
+       p->first_ack, p->packet_type);
+  if (p->packet_type == PACKET_TYPE_HANDSHAKE) {
+    DLOG("cookie epoch=%u, cookie=0x", p->data.handshake.cookie_epoch);
+    debug_print_hex(p->data.handshake.cookie, sizeof(p->data.handshake.cookie));
+  } else {
+    DLOG("Acks:\n");
+    for (uint32_t i = 0; i < ARRAY_LEN(p->data.acks); i++) {
+      uint32_t acki = (p->first_ack + i) % ARRAY_LEN(p->data.acks);
+      uint32_t ackid = ntohl(p->data.acks[acki].id);
+      if (!ackid) continue;  // empty slot
+      DLOG(" acki=%d id=%d rxtime=%u\n",
+           acki, ackid, ntohl(p->data.acks[acki].rxtime));
+    }
+  }
+}
 
 void prepare_tx_packet(struct Session *s) {
   s->tx.magic = htonl(MAGIC);
@@ -222,35 +398,304 @@
   s->tx.clockdiff = s->start_rtxtime ?
       htonl(s->start_rxtime - s->start_rtxtime) : 0;
   s->tx.num_lost = htonl(s->num_lost);
-  s->tx.first_ack = htonl(s->next_txack_index);
+  s->tx.first_ack = s->next_txack_index;
+  switch (s->handshake_state) {
+    case Session::NEW_SESSION:
+    case Session::HANDSHAKE_REQUESTED:
+    case Session::COOKIE_GENERATED:
+      DLOG("prepare_tx_packet: Sending handshake packet\n");
+      s->tx.packet_type = PACKET_TYPE_HANDSHAKE;
+      break;
+    case Session::ESTABLISHED:
+      s->tx.packet_type = PACKET_TYPE_ACK;
+      break;
+    default:
+      fprintf(stderr, "Unknown handshake state %d\n", s->handshake_state);
+      exit(2);
+  }
+  // note: tx.data.acks[] is filled in incrementally; we just transmit the
+  // current state of it here.  The reason we keep a list of the most recent
+  // acks is in case our packet gets lost, so the receiver will have more
+  // chances to receive the timing information for the packets it sent us.
+  debug_print_packet(&s->tx);
 }
 
-static int send_packet(struct Session *s,
-                       int sock,
-                       struct sockaddr *remoteaddr,
-                       socklen_t remoteaddr_len) {
-  // note: tx.acks[] is filled in incrementally; we just transmit the current
-  // state of it here.  The reason we keep a list of the most recent acks is in
-  // case our packet gets lost, so the receiver will have more chances to
-  // receive the timing information for the packets it sent us.
+
+void prepare_handshake_reply_packet(Packet *tx, Packet *rx, uint32_t now) {
+  memset(tx, 0, sizeof(*tx));
+  tx->magic = htonl(MAGIC);
+  tx->id = rx->id;
+  // TODO(pmccurdy): Establish limits on the allowed usec_per_pkt values here
+  tx->usec_per_pkt = rx->usec_per_pkt;
+  tx->txtime = now;
+  tx->clockdiff = htonl(now - ntohl(rx->txtime));
+  tx->num_lost = htonl(0);
+  tx->packet_type = PACKET_TYPE_HANDSHAKE;
+}
+
+
+int send_packet(struct Session *s, int sock, int is_server) {
   if (is_server) {
     if (sendto(sock, &s->tx, sizeof(s->tx), 0,
-               remoteaddr, remoteaddr_len) < 0) {
+               (struct sockaddr *)&s->remoteaddr, s->remoteaddr_len) < 0) {
       perror("sendto");
     }
   } else {
+    DLOG("Calling send on socket %d, size=%ld\n", sock, sizeof(s->tx));
     if (send(sock, &s->tx, sizeof(s->tx), 0) < 0) {
       int e = errno;
       perror("send");
       if (e == ECONNREFUSED) return 2;
     }
   }
-  s->next_send += s->usec_per_pkt;
+  if (is_server ||
+      s->handshake_state == Session::ESTABLISHED ||
+      s->handshake_state == Session::COOKIE_GENERATED) {
+    DLOG("send_packet: ack packet, next_send in %d (from %d to %d)\n",
+         s->usec_per_pkt, s->next_send, s->next_send + s->usec_per_pkt);
+    s->next_send += s->usec_per_pkt;
+  } else {
+    // Handle resending handshake packets from the client.  If they get lost
+    // before we get a valid cookie from the server, the server won't know about
+    // us, and our normal retry procedure would get us out of sync.
+    if (s->handshake_state == Session::NEW_SESSION) {
+      DLOG("Handshake state: sending handshake packet, moving to "
+           "HANDSHAKE_REQUESTED\n");
+      s->handshake_state = Session::HANDSHAKE_REQUESTED;
+      s->handshake_retry_count = 0;
+    } else {
+      s->handshake_retry_count++;
+    }
+    // Limit the backoff to a factor of 2^10.
+    uint32_t timeout = Session::handshake_timeout_usec *
+                       (1 << std::min(10, s->handshake_retry_count));
+    DLOG("Sending handshake, retries=%d, next_send in %d us (from %u to %u)\n",
+         s->handshake_retry_count, timeout, s->next_send,
+         s->next_send + timeout);
+    s->next_send += timeout;
+    // Don't count the handshake packet as part of the sequence.
+    s->next_tx_id--;
+  }
   return 0;
 }
 
+int send_waiting_packets(Sessions *sessions, int sock, uint32_t now,
+                         int is_server) {
+  if (sessions == NULL) {
+    return -1;
+  }
+  // TODO(pmccurdy): This will incorrectly send packets too early during the
+  // time period where now + usec_per_pkt wraps around, i.e. about once per 71
+  // minutes, and will end up blasting packets as fast as possible for that
+  // duration.  Consider calculating next_send and now as 64-bit values, and
+  // truncating to 32 bits when transmitting.
+  while (sessions->next_sends.size() > 0 &&
+         DIFF(now, sessions->next_send_time()) >= 0) {
+    SessionMap::iterator it = sessions->next_sends.top();
+    sessions->next_sends.pop();
+    Session &s = it->second;
+    prepare_tx_packet(&s);
+    int err = send_packet(&s, sock, is_server);
+    if (err != 0) {
+      return err;
+    }
+    // TODO(pmccurdy): Detect connection refused on a per-client basis.  Use
+    // recvmsg with the MSG_ERRQUEUE flag to get error and client address,
+    // instead of waiting for timeout.
+    // TODO(pmccurdy): Support very low packet-per-second values, e.g. one
+    // packet per hour, without constantly disconnecting the client.
+    // TODO(pmccurdy): Instead of a fixed timeout, evict clients if they miss
+    // a certain number of expected transmissions, that number changing based on
+    // the number of packets they've already sent.
+    if (is_server && DIFF(now, s.last_rxtime) > 60 * 1000 * 1000) {
+      fprintf(stderr, "client %s disconnected.\n",
+              sockaddr_to_str((struct sockaddr *)&s.remoteaddr));
+      sessions->session_map.erase(s.remoteaddr);
+    } else {
+      sessions->next_sends.push(it);
+    }
+  }
+  return 0;
+}
 
-void handle_packet(struct Session *s, uint32_t now) {
+int read_incoming_packet(Sessions *s, int sock, uint32_t now, int is_server) {
+  struct sockaddr_storage rxaddr;
+  socklen_t rxaddr_len = sizeof(rxaddr);
+
+  Packet rx;
+  ssize_t got = recvfrom(sock, &rx, sizeof(rx), 0,
+                         (struct sockaddr *)&rxaddr, &rxaddr_len);
+  if (got < 0) {
+    int e = errno;
+    perror("recvfrom");
+    return e;
+  }
+  if (got != sizeof(rx) || rx.magic != htonl(MAGIC)) {
+    fprintf(stderr, "got invalid packet of length %ld, magic=%d from %s\n",
+            (long)got, ntohl(rx.magic),
+            sockaddr_to_str((struct sockaddr *)&rxaddr));
+    return EINVAL;
+  }
+  switch (rx.packet_type) {
+    case PACKET_TYPE_HANDSHAKE:
+    case PACKET_TYPE_ACK:
+      break;
+    default:
+      fprintf(stderr, "received unknown packet type %d\n", rx.packet_type);
+      return EINVAL;
+  }
+
+  Session *session = NULL;
+  if (is_server) {
+    SessionMap::iterator it = s->session_map.find(rxaddr);
+    if (it != s->session_map.end()) {
+      session = &it->second;
+    } else {
+      // Note: we don't want to allocate any memory here until the client has
+      // completed the handshake.
+      if (rx.packet_type != PACKET_TYPE_HANDSHAKE) {
+        fprintf(stderr, "Received non-handshake packet from unknown client\n");
+        // TODO(pmccurdy): Reply with a new handshake packet, including a
+        // cookie; we may have dropped a legit client and we need to tell them
+        // to renegotiate.
+        return -1;
+      }
+    }
+  } else {
+    SessionMap::iterator it = s->session_map.begin();
+    if (it == s->session_map.end()) {
+      fprintf(stderr, "No session configured for %s when receiving packet\n",
+              sockaddr_to_str((struct sockaddr *)&rxaddr));
+      return EINVAL;
+    }
+    DLOG("read_incoming_packet: Client received %s packet from server\n",
+         rx.packet_type == PACKET_TYPE_ACK ? "ack" : "handshake");
+    session = &it->second;
+  }
+  handle_packet(s, session, &rx, sock, &rxaddr, rxaddr_len, now, is_server);
+
+  return 0;
+}
+
+// Checks what kind of packet we've received, and processes it appropriately.
+// Session may be null if we're dealing with a handshake packet for a new
+// connection.
+void handle_packet(struct Sessions *s, struct Session *session, Packet *rx,
+                   int sock, struct sockaddr_storage *rxaddr,
+                   socklen_t rxaddr_len, uint32_t now, int is_server) {
+  switch (rx->packet_type) {
+    case PACKET_TYPE_HANDSHAKE:
+      if (is_server) {
+        handle_new_client_handshake_packet(s, rx, sock, rxaddr, rxaddr_len,
+                                           now);
+        return;
+      } else {
+        DLOG("Client received handshake packet from server\n");
+        handle_server_handshake_packet(s, rx, now);
+        return;
+      }
+      break;
+    case PACKET_TYPE_ACK:
+      if (session != NULL) {
+        memcpy(&session->rx, rx, sizeof(session->rx));
+        if (!is_server &&
+            session->handshake_state == Session::COOKIE_GENERATED) {
+          // Now we know the server has accepted our connection.  Clear out the
+          // handshake data from the send buffer and prepare to track acks.
+          DLOG("Ack from server on new connection; moving to state "
+               "ESTABLISHED.");
+          session->handshake_state = Session::ESTABLISHED;
+          memset(&session->tx.data.acks, 0, sizeof(session->tx.data.acks));
+        }
+      }
+      handle_ack_packet(session, now);
+      break;
+    default:
+      fprintf(stderr, "handle_packet called for unknown packet type %d\n",
+              rx->packet_type);
+      break;
+  }
+}
+
+void handle_new_client_handshake_packet(Sessions *s, Packet *rx, int sock,
+                                       struct sockaddr_storage *remoteaddr,
+                                       size_t remoteaddr_len, uint32_t now) {
+  assert(s != NULL);
+  assert(rx != NULL);
+  assert(remoteaddr != NULL);
+  DLOG("Server received handshake packet from client; cookie epoch=%u\n",
+       rx->data.handshake.cookie_epoch);
+  if (rx->data.handshake.cookie_epoch == 0) {
+    // New connection with no cookie.  Return a cookie to validate the client.
+    s->session_map.erase(*remoteaddr);
+    fprintf(stderr, "New connection from %s, sending cookie\n",
+            sockaddr_to_str((struct sockaddr *)remoteaddr));
+    Packet tx;
+    memset(&tx, 0, sizeof(tx));
+    prepare_handshake_reply_packet(&tx, rx, now);
+    s->CalculateCookie(&tx, remoteaddr, remoteaddr_len);
+    sendto(sock, &tx, sizeof(tx), 0, (struct sockaddr *)remoteaddr,
+           remoteaddr_len);
+    // The handshake_state is conceptually in the COOKIE_GENERATED state now,
+    // but the whole point of the cookie is to avoid saving state in the server,
+    // so we don't store a Session here.
+  } else {
+    // Cookie provided, validate it to accept or reject the connection.
+    if (!s->ValidateCookie(rx, remoteaddr, remoteaddr_len)) {
+      return;
+    }
+    fprintf(stderr, "New client connection: %s\n",
+            sockaddr_to_str((struct sockaddr *)remoteaddr));
+    SessionMap::iterator it = s->NewSession(
+        now + 10 * 1000, ntohl(rx->usec_per_pkt), remoteaddr, remoteaddr_len);
+    Session &session = it->second;
+    session.handshake_state = Session::ESTABLISHED;
+    memcpy(&session.rx, rx, sizeof(session.rx));
+    // This is a new session we haven't sent any timing packets on, so the
+    // client can't possibly have acknowledged any packets.  Replace the
+    // handshake data with a set of empty acks and process as normal.
+    session.rx.packet_type = PACKET_TYPE_ACK;
+    memset(&session.rx.data.acks, 0, sizeof(session.rx.data.acks));
+    assert(sizeof(session.rx.data.acks) > sizeof(void *));
+    handle_ack_packet(&session, now);
+  }
+}
+
+void handle_server_handshake_packet(Sessions *s, Packet *rx, uint32_t now) {
+  assert(s != NULL);
+  assert(rx != NULL);
+  assert(s->session_map.size() == 1);
+  assert(s->next_sends.size() == 1);
+
+  SessionMap::iterator it = s->session_map.begin();
+  Session &session = it->second;
+  // We don't need to resend the handshake packet any more.
+  s->next_sends.pop();
+
+  session.tx.packet_type = PACKET_TYPE_HANDSHAKE;
+  session.tx.data.handshake.cookie_epoch = rx->data.handshake.cookie_epoch;
+  memcpy(&session.tx.data.handshake.cookie, &rx->data.handshake.cookie,
+         COOKIE_SIZE);
+  int usec_per_pkt = ntohl(rx->usec_per_pkt);
+  if (usec_per_pkt != session.usec_per_pkt) {
+    fprintf(stderr, "Server overrode packets per second to %f\n",
+            1000000.0 / usec_per_pkt);
+    session.usec_per_pkt = usec_per_pkt;
+  }
+  DLOG("Handshake state: client received cookie from server, moving to "
+       "COOKIE_GENERATED; next_send=%d (was %d)\n",
+       now, session.next_send);
+  DLOG("Handshake: cookie epoch=%d, cookie=0x",
+       rx->data.handshake.cookie_epoch);
+  debug_print_hex(rx->data.handshake.cookie, sizeof(rx->data.handshake.cookie));
+  session.handshake_state = Session::COOKIE_GENERATED;
+  session.next_send = now;
+  s->next_sends.push(it);
+}
+
+void handle_ack_packet(struct Session *s, uint32_t now) {
+  assert(s != NULL);
+  assert(s->rx.packet_type == PACKET_TYPE_ACK);
   // process the incoming packet header.
   // Most of the complexity here comes from the fact that the remote
   // system's clock will be skewed vs. ours.  (We use CLOCK_MONOTONIC
@@ -313,6 +758,8 @@
     s->start_rxtime = rxtime - id * s->usec_per_pkt;
   }
   int32_t rxdiff = DIFF(rxtime, s->start_rxtime + id * s->usec_per_pkt);
+  DLOG("ack: rxdiff=%d, rxtime=%u, start_rxtime=%u, id=%d, usec_per_pkt=%d\n",
+       rxdiff, rxtime, s->start_rxtime, id, s->usec_per_pkt);
 
   // Figure out the offset between our clock and the remote's clock, so we can
   // calculate the minimum round trip time (rtt). Then, because the consecutive
@@ -365,6 +812,8 @@
     s->lat_rx_sum += s->lat_rx;
     s->lat_rx_var_sum += s->lat_rx * s->lat_rx;
   }
+  DLOG("ack packet: rx id=%d, clockdiff=%d, rtt=%d, offset=%d, rxdiff=%d\n",
+       id, clockdiff, rtt, offset, rxdiff);
 
   // Note: the way ok_to_print is structured, if there is a dropout in the
   // connection for more than usec_per_print, we will statistically end up
@@ -398,25 +847,28 @@
   }
 
   // schedule this for an ack next time we send the packet
-  s->tx.acks[s->next_txack_index].id = htonl(id);
-  s->tx.acks[s->next_txack_index].rxtime = htonl(rxtime);
-  s->next_txack_index = (s->next_txack_index + 1) % ARRAY_LEN(s->tx.acks);
+  s->tx.data.acks[s->next_txack_index].id = htonl(id);
+  s->tx.data.acks[s->next_txack_index].rxtime = htonl(rxtime);
+  s->next_txack_index = (s->next_txack_index + 1) % ARRAY_LEN(s->tx.data.acks);
 
   // see which of our own transmitted packets have been acked
-  uint32_t first_ack = ntohl(s->rx.first_ack);
-  for (uint32_t i = 0; i < ARRAY_LEN(s->rx.acks); i++) {
-    uint32_t acki = (first_ack + i) % ARRAY_LEN(s->rx.acks);
-    uint32_t ackid = ntohl(s->rx.acks[acki].id);
+  uint32_t first_ack = s->rx.first_ack;
+  for (uint32_t i = 0; i < ARRAY_LEN(s->rx.data.acks); i++) {
+    uint32_t acki = (first_ack + i) % ARRAY_LEN(s->rx.data.acks);
+    uint32_t ackid = ntohl(s->rx.data.acks[acki].id);
     if (!ackid) continue;  // empty slot
     if (DIFF(ackid, s->next_rxack_id) >= 0) {
       // an expected ack
       uint32_t start_txtime = s->next_send - s->next_tx_id * s->usec_per_pkt;
       uint32_t txtime = start_txtime + ackid * s->usec_per_pkt;
-      uint32_t rrxtime = ntohl(s->rx.acks[acki].rxtime);
+      uint32_t rrxtime = ntohl(s->rx.data.acks[acki].rxtime);
       uint32_t rxtime = rrxtime + offset;
       // note: already contains 1/2 rtt, unlike rxdiff
       int32_t txdiff = DIFF(rxtime, txtime);
-      if (s->usec_per_print <= 0 && s->last_ackinfo[0]) {
+      DLOG("acki=%d ackid=%d txdiff=%d rxtime=%u txtime=%u offset=%u, "
+           "start_txtime=%u\n",
+           acki, ackid, txdiff, rxtime, txtime, offset, start_txtime);
+      if (!quiet && s->usec_per_print <= 0 && s->last_ackinfo[0]) {
         // only print multiple acks per rx if no usec_per_print limit
         if (want_timestamps) print_timestamp(rxtime);
         printf("%12s\n", s->last_ackinfo);
@@ -439,11 +891,8 @@
   s->last_rxtime = rxtime;
 }
 
-
 int isoping_main(int argc, char **argv) {
-  struct sockaddr_in6 listenaddr, rxaddr, last_rxaddr;
-  struct sockaddr *remoteaddr = NULL;
-  socklen_t remoteaddr_len = 0, rxaddr_len = 0;
+  struct sockaddr_in6 listenaddr;
   struct addrinfo *ai = NULL;
   int sock = -1;
 
@@ -494,6 +943,10 @@
     return 1;
   }
 
+  int is_server;
+  uint32_t now = ustime();       // current time
+  Sessions sessions;
+
   if (argc - optind == 0) {
     is_server = 1;
     memset(&listenaddr, 0, sizeof(listenaddr));
@@ -529,8 +982,8 @@
       perror("connect");
       return 1;
     }
-    remoteaddr = ai->ai_addr;
-    remoteaddr_len = ai->ai_addrlen;
+    sessions.NewSession(now, 1e6 / packets_per_sec,
+                        (struct sockaddr_storage *)ai->ai_addr, ai->ai_addrlen);
   } else {
     usage_and_die(argv[0]);
   }
@@ -553,15 +1006,13 @@
     }
   }
 
-  uint32_t now = ustime();       // current time
-
   struct sigaction act;
   memset(&act, 0, sizeof(act));
   act.sa_handler = sighandler;
   act.sa_flags = SA_RESETHAND;
   sigaction(SIGINT, &act, NULL);
 
-  struct Session s(now);
+  uint32_t last_secret_update_time = 0;
 
   while (!want_to_die) {
     fd_set rfds;
@@ -571,91 +1022,59 @@
     tv.tv_sec = 0;
 
     now = ustime();
-    if (DIFF(s.next_send, now) < 0) {
+    if (sessions.next_sends.size() == 0 ||
+        DIFF(sessions.next_send_time(), now) < 0) {
       tv.tv_usec = 0;
     } else {
-      tv.tv_usec = DIFF(s.next_send, now);
+      tv.tv_usec = DIFF(sessions.next_send_time(), now);
     }
-    int nfds = select(sock + 1, &rfds, NULL, NULL, remoteaddr ? &tv : NULL);
+    int nfds = select(sock + 1, &rfds, NULL, NULL,
+                      sessions.next_sends.size() > 0 ? &tv : NULL);
     now = ustime();
     if (nfds < 0 && errno != EINTR) {
       perror("select");
       return 1;
     }
 
-    // time to send the next packet?
-    if (remoteaddr && DIFF(now, s.next_send) >= 0) {
-      prepare_tx_packet(&s);
-      int err = send_packet(&s, sock, remoteaddr, remoteaddr_len);
-      if (err != 0) {
-        return err;
-      }
-      // TODO(pmccurdy): Track disconnections across multiple clients.  Use
-      // recvmsg with the MSG_ERRQUEUE flag to detect connection refused.
-      if (is_server && DIFF(now, s.last_rxtime) > 60*1000*1000) {
-        fprintf(stderr, "client disconnected.\n");
-        remoteaddr = NULL;
-      }
+    // Periodically check if the cookie secrets need updating.
+    if (is_server && (now - last_secret_update_time) > 1000000) {
+      sessions.MaybeRotateCookieSecrets();
+      last_secret_update_time = now;
+    }
+
+    int err = send_waiting_packets(&sessions, sock, now, is_server);
+    if (err != 0) {
+      return err;
     }
 
     if (nfds > 0) {
-      // incoming packet
-      rxaddr_len = sizeof(rxaddr);
-      ssize_t got = recvfrom(sock, &s.rx, sizeof(s.rx), 0,
-                             (struct sockaddr *)&rxaddr, &rxaddr_len);
-      if (got < 0) {
-        int e = errno;
-        perror("recvfrom");
-        if (!is_server && e == ECONNREFUSED) return 2;
+      err = read_incoming_packet(&sessions, sock, now, is_server);
+      if (!is_server && err == ECONNREFUSED) return 2;
+      if (err != 0) {
         continue;
       }
-      if (got != sizeof(s.rx) || s.rx.magic != htonl(MAGIC)) {
-        fprintf(stderr, "got invalid packet of length %ld\n", (long)got);
-        continue;
-      }
-
-      // is it a new client?
-      if (is_server) {
-        // TODO(pmccurdy): Maintain a hash table of Sessions, look up based
-        // on rxaddr, create a new one if necessary, remove this resetting code.
-        if (!remoteaddr ||
-            memcmp(&rxaddr, &last_rxaddr, sizeof(rxaddr)) != 0) {
-          fprintf(stderr, "new client connected: %s\n",
-                  sockaddr_to_str((struct sockaddr *)&rxaddr));
-          memcpy(&last_rxaddr, &rxaddr, sizeof(rxaddr));
-          remoteaddr = (struct sockaddr *)&last_rxaddr;
-          remoteaddr_len = rxaddr_len;
-
-          s.next_send = now + 10*1000;
-          s.next_tx_id = 1;
-          s.next_rx_id = s.next_rxack_id = 0;
-          s.start_rtxtime = s.start_rxtime = 0;
-          s.num_lost = 0;
-          s.next_txack_index = 0;
-          s.usec_per_pkt = ntohl(s.rx.usec_per_pkt);
-          memset(&s.tx, 0, sizeof(s.tx));
-        }
-      }
-
-      handle_packet(&s, now);
     }
   }
 
-  // TODO(pmccurdy): Separate out per-client and global stats.
-  printf("\n---\n");
-  printf("tx: min/avg/max/mdev = %.2f/%.2f/%.2f/%.2f ms\n",
-         s.lat_tx_min / 1000.0,
-         DIV(s.lat_tx_sum, s.lat_tx_count) / 1000.0,
-         s.lat_tx_max / 1000.0,
-         onepass_stddev(
-             s.lat_tx_var_sum, s.lat_tx_sum, s.lat_tx_count) / 1000.0);
-  printf("rx: min/avg/max/mdev = %.2f/%.2f/%.2f/%.2f ms\n",
-         s.lat_rx_min / 1000.0,
-         DIV(s.lat_rx_sum, s.lat_rx_count) / 1000.0,
-         s.lat_rx_max / 1000.0,
-         onepass_stddev(
-             s.lat_rx_var_sum, s.lat_rx_sum, s.lat_rx_count) / 1000.0);
-  printf("\n");
+  // TODO(pmccurdy): Separate out per-client and global stats, print stats for
+  // the server when each client disconnects.
+  if (!is_server) {
+    Session &s = sessions.session_map.begin()->second;
+    printf("\n---\n");
+    printf("tx: min/avg/max/mdev = %.2f/%.2f/%.2f/%.2f ms\n",
+           s.lat_tx_min / 1000.0,
+           DIV(s.lat_tx_sum, s.lat_tx_count) / 1000.0,
+           s.lat_tx_max / 1000.0,
+           onepass_stddev(
+               s.lat_tx_var_sum, s.lat_tx_sum, s.lat_tx_count) / 1000.0);
+    printf("rx: min/avg/max/mdev = %.2f/%.2f/%.2f/%.2f ms\n",
+           s.lat_rx_min / 1000.0,
+           DIV(s.lat_rx_sum, s.lat_rx_count) / 1000.0,
+           s.lat_rx_max / 1000.0,
+           onepass_stddev(
+               s.lat_rx_var_sum, s.lat_rx_sum, s.lat_rx_count) / 1000.0);
+    printf("\n");
+  }
 
   if (ai) freeaddrinfo(ai);
   if (sock >= 0) close(sock);
diff --git a/cmds/isoping.h b/cmds/isoping.h
index 52fcccb..9c2ee85 100644
--- a/cmds/isoping.h
+++ b/cmds/isoping.h
@@ -16,7 +16,24 @@
 #ifndef ISOPING_H
 #define ISOPING_H
 
+#include <map>
+#include <netinet/in.h>
+#include <openssl/evp.h>
+#include <queue>
+#include <random>
 #include <stdint.h>
+#include <string.h>
+#include <sys/socket.h>
+
+// Number of bytes required to store the cookie, which is a SHA-256 hash.
+#define COOKIE_SIZE 32
+// Number of bytes used to store the random cookie secret.
+#define COOKIE_SECRET_SIZE 16
+
+enum {
+  PACKET_TYPE_ACK = 0,
+  PACKET_TYPE_HANDSHAKE,
+};
 
 // Layout of the UDP packets exchanged between client and server.
 // All integers are in network byte order.
@@ -28,21 +45,45 @@
   uint32_t clockdiff; // estimate of (transmitter's clk) - (receiver's clk)
   uint32_t usec_per_pkt; // microseconds of delay between packets
   uint32_t num_lost;  // number of pkts transmitter expected to get but didn't
-  uint32_t first_ack; // starting index in acks[] circular buffer
-  struct {
-    // txtime==0 for empty elements in this array.
-    uint32_t id;      // id field from a received packet
-    uint32_t rxtime;  // receiver's monotonic time when pkt arrived
-  } acks[64];
+  uint8_t packet_type; // 0 for acks, 1 for handshake packet
+  uint8_t first_ack;  // starting index in acks[] circular buffer
+  union {
+    // Data used for handshake packets.
+    struct {
+      uint32_t version; // max version of the isoping protocol supported
+      uint32_t cookie_epoch; // which cookie we're using
+      unsigned char cookie[COOKIE_SIZE]; // actual cookie value
+    } handshake;
+    // Data used for ack packets.
+    struct {
+      // txtime==0 for empty elements in this array.
+      uint32_t id;      // id field from a received packet
+      uint32_t rxtime;  // receiver's monotonic time when pkt arrived
+    } acks[64];
+  } data;
 };
 
 
 // Data we track per session.
 struct Session {
-  Session(uint32_t now);
+  Session(uint32_t first_send, uint32_t usec_per_pkt,
+          const struct sockaddr_storage &remoteaddr, size_t remoteaddr_len);
   int32_t usec_per_pkt;
   int32_t usec_per_print;
 
+  // The peer's address.
+  struct sockaddr_storage remoteaddr;
+  socklen_t remoteaddr_len;
+
+  enum {
+    NEW_SESSION = 0,     // No packets exchanged yet.
+    HANDSHAKE_REQUESTED, // Client has sent initial packet to server, i.e. SYN.
+    COOKIE_GENERATED,    // Server has replied with cookie, i.e. SYN|ACK.
+    ESTABLISHED          // Client has echoed cookie back, i.e. ACK.
+  } handshake_state;
+  int handshake_retry_count;
+  static const int handshake_timeout_usec = 1000000;
+
   // WARNING: lots of math below relies on well-defined uint32/int32
   // arithmetic overflow behaviour, plus the fact that when we subtract
   // two successive timestamps (for example) they will be less than 2^31
@@ -70,12 +111,122 @@
       lat_rx_count, lat_rx_sum, lat_rx_var_sum;
 };
 
-// Process the Session's incoming packet, from s->rx.
-void handle_packet(struct Session *s, uint32_t now);
+// Comparator for use with sockaddr_in and sockaddr_in6 values.  Sorts by
+// address family first, then on the IPv4/6 address, then the port number.
+struct CompareSockaddr {
+  bool operator()(const struct sockaddr_storage &lhs,
+                  const struct sockaddr_storage &rhs);
+};
+
+typedef std::map<struct sockaddr_storage, Session, CompareSockaddr>
+    SessionMap;
+
+// Compares the next_send values of each referenced Session, sorting the earlier
+// timestamps first.
+struct CompareNextSend {
+  bool operator()(const SessionMap::iterator &lhs,
+                  const SessionMap::iterator &rhs);
+};
+
+struct Sessions {
+ public:
+  Sessions()
+      : md(EVP_sha256()),
+        rng(std::random_device()()),
+        cookie_epoch(0) {
+    NewRandomCookieSecret();
+    EVP_MD_CTX_init(&digest_context);
+  }
+
+  ~Sessions() {
+    EVP_MD_CTX_cleanup(&digest_context);
+  }
+
+  // Rotates the cookie secrets if they haven't been changed in a while.
+  void MaybeRotateCookieSecrets();
+
+  // Rotate the cookie secrets using the given epoch directly.  Only for use in
+  // unit tests.
+  void RotateCookieSecrets(uint32_t new_epoch);
+
+  // Calculates a handshake cookie based on the provided client IP address and
+  // the relevant parameters in p, using the current cookie secret, and places
+  // the result in p.
+  bool CalculateCookie(Packet *p, struct sockaddr_storage *remoteaddr,
+                       size_t remoteaddr_len);
+
+  // Returns true if the packet contains a handshake packet with a valid cookie.
+  bool ValidateCookie(Packet *p, struct sockaddr_storage *addr,
+                      socklen_t addr_len);
+
+  SessionMap::iterator NewSession(uint32_t first_send,
+                                  uint32_t usec_per_pkt,
+                                  struct sockaddr_storage *addr,
+                                  socklen_t addr_len);
+
+  uint32_t next_send_time() {
+    if (next_sends.size() == 0) {
+      return 0;
+    }
+    return next_sends.top()->second.next_send;
+  }
+
+  // All active sessions, indexed by remote address/port.
+  SessionMap session_map;
+  // A queue of upcoming send times, ordered most recent first, referencing
+  // entries in the session map.
+  std::priority_queue<SessionMap::iterator, std::vector<SessionMap::iterator>,
+      CompareNextSend> next_sends;
+
+ private:
+  void NewRandomCookieSecret();
+  bool CalculateCookieWithSecret(Packet *p, struct sockaddr_storage *remoteaddr,
+                                 size_t remoteaddr_len, unsigned char *secret,
+                                 size_t secret_len);
+
+  // Fields required for calculating and verifying cookies.
+  EVP_MD_CTX digest_context;
+  const EVP_MD *md;
+  std::mt19937_64 rng;
+  uint32_t cookie_epoch;
+  unsigned char cookie_secret[COOKIE_SECRET_SIZE];
+  uint32_t prev_cookie_epoch;
+  unsigned char prev_cookie_secret[COOKIE_SECRET_SIZE];
+};
+
+// Process an incoming packet from the socket.
+void handle_packet(struct Sessions *s, struct Session *session, Packet *rx,
+                   int sock, struct sockaddr_storage *rxaddr,
+                   socklen_t rxaddr_len, uint32_t now, int is_server);
+
+// Process an established Session's incoming ack packet, from s->rx.
+void handle_ack_packet(struct Session *s, uint32_t now);
+
+// Server-only: processes a handshake packet from a new client in rx. Replies
+// with a cookie if no cookie provided, or validates the provided cookie and
+// establishes a new Session.
+void handle_new_client_handshake_packet(Sessions *s, Packet *rx, int sock,
+                                       struct sockaddr_storage *remoteaddr,
+                                       size_t remoteaddr_len, uint32_t now);
+
+// Client-only: processes a handshake packet received from the server.
+// Configures the Session to echo the provided cookie back to the server.
+void handle_server_handshake_packet(Sessions *s, Packet *rx, uint32_t now);
 
 // Sets all the elements of s->tx to be ready to be sent to the other side.
 void prepare_tx_packet(struct Session *s);
 
+// Sends a packet to all waiting sessions where the appropriate amount of time
+// has passed.
+int send_waiting_packets(Sessions *s, int sock, uint32_t now, int is_server);
+
+// Sends a packet from the given session to the given socket immediately.
+int send_packet(struct Session *s, int sock, int is_server);
+
+// Reads a packet from sock and stores it in s->rx.  Assumes a packet is
+// currently readable.
+int read_incoming_packet(Sessions *s, int sock, uint32_t now, int is_server);
+
 // Parses arguments and runs the main loop.  Distinct from main() for unit test
 // purposes.
 int isoping_main(int argc, char **argv);
diff --git a/cmds/isoping_test.cc b/cmds/isoping_test.cc
index 84edc46..ca0c440 100644
--- a/cmds/isoping_test.cc
+++ b/cmds/isoping_test.cc
@@ -15,21 +15,26 @@
  */
 
 #include <arpa/inet.h>
+#include <errno.h>
 #include <limits.h>
+#include <memory.h>
 #include <stdio.h>
-
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <unistd.h>
 #include <wvtest.h>
 
 #include "isoping.h"
 
-uint32_t send_next_packet(Session *from, uint32_t from_base,
+uint32_t send_next_ack_packet(Session *from, uint32_t from_base,
                           Session *to, uint32_t to_base, uint32_t latency) {
   uint32_t t = from->next_send - from_base;
   prepare_tx_packet(from);
   to->rx = from->tx;
   from->next_send += from->usec_per_pkt;
   t += latency;
-  handle_packet(to, to_base + t);
+  handle_ack_packet(to, to_base + t);
   fprintf(stderr,
           "**Sent packet: txtime=%d, start_txtime=%d, rxtime=%d, "
           "start_rxtime=%d, latency=%d, t_from=%d, t_to=%d\n",
@@ -52,10 +57,14 @@
   uint32_t cbase = 400 * 1000;
   uint32_t sbase = 600 * 1000;
   uint32_t real_clockdiff = sbase - cbase;
+  uint32_t usec_per_pkt = 100 * 1000;
 
   // The states of the client and server.
-  struct Session c(cbase);
-  struct Session s(sbase);
+  struct sockaddr_storage empty_sockaddr;
+  struct Session c(cbase, usec_per_pkt, empty_sockaddr, sizeof(empty_sockaddr));
+  struct Session s(sbase, usec_per_pkt, empty_sockaddr, sizeof(empty_sockaddr));
+  c.handshake_state = Session::ESTABLISHED;
+  s.handshake_state = Session::ESTABLISHED;
 
   // One-way latencies: cs_latency is the latency from client to server;
   // sc_latency is from server to client.
@@ -68,10 +77,7 @@
 
   // Send the initial packet from client to server.  This isn't enough to let us
   // draw any useful latency conclusions.
-  // TODO(pmccurdy): Setting next_send is duplicating some work done in the main
-  // loop / send_packet.  Extract that into somewhere testable, then test it.
-  c.next_send = cbase;
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency);
+  t = send_next_ack_packet(&c, cbase, &s, sbase, cs_latency);
   uint32_t rxtime = sbase + t;
   s.next_send = rxtime + 10 * 1000;
 
@@ -80,16 +86,16 @@
   WVPASSEQ(s.rx.clockdiff, 0);
   WVPASSEQ(s.last_rxtime, rxtime);
   WVPASSEQ(s.min_cycle_rxdiff, 0);
-  WVPASSEQ(ntohl(s.tx.acks[0].id), 1);
+  WVPASSEQ(ntohl(s.tx.data.acks[0].id), 1);
   WVPASSEQ(s.next_txack_index, 1);
-  WVPASSEQ(ntohl(s.tx.acks[ntohl(s.tx.first_ack)].id), 1);
-  WVPASSEQ(ntohl(s.tx.acks[ntohl(s.tx.first_ack)].rxtime), rxtime);
+  WVPASSEQ(ntohl(s.tx.data.acks[s.tx.first_ack].id), 1);
+  WVPASSEQ(ntohl(s.tx.data.acks[s.tx.first_ack].rxtime), rxtime);
   WVPASSEQ(s.start_rxtime, rxtime - c.usec_per_pkt);
   WVPASSEQ(s.start_rtxtime, cbase - c.usec_per_pkt);
   WVPASSEQ(s.next_send, rxtime + 10 * 1000);
 
   // Reply to the client.
-  t = send_next_packet(&s, sbase, &c, cbase, sc_latency);
+  t = send_next_ack_packet(&s, sbase, &c, cbase, sc_latency);
 
   // Now we have enough data to figure out latencies on the client.
   rxtime = cbase + t;
@@ -97,8 +103,8 @@
   WVPASSEQ(c.start_rtxtime, sbase + cs_latency + 10 * 1000 - s.usec_per_pkt);
   WVPASSEQ(c.min_cycle_rxdiff, 0);
   WVPASSEQ(ntohl(c.rx.clockdiff), sbase - cbase + cs_latency);
-  WVPASSEQ(ntohl(c.tx.acks[ntohl(c.tx.first_ack)].id), 1);
-  WVPASSEQ(ntohl(c.tx.acks[ntohl(c.tx.first_ack)].rxtime), rxtime);
+  WVPASSEQ(ntohl(c.tx.data.acks[c.tx.first_ack].id), 1);
+  WVPASSEQ(ntohl(c.tx.data.acks[c.tx.first_ack].rxtime), rxtime);
   WVPASSEQ(c.num_lost, 0);
   WVPASSEQ(c.lat_tx_count, 1);
   WVPASSEQ(c.lat_tx, half_rtt);
@@ -107,15 +113,15 @@
   WVPASSEQ(c.num_lost, 0);
 
   // Round 2
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency);
+  t = send_next_ack_packet(&c, cbase, &s, sbase, cs_latency);
   rxtime = sbase + t;
 
   // Now the server also knows latencies.
   WVPASSEQ(s.start_rxtime, sbase + cs_latency - s.usec_per_pkt);
   WVPASSEQ(s.start_rtxtime, cbase - c.usec_per_pkt);
   WVPASSEQ(ntohl(s.rx.clockdiff), cbase - sbase + sc_latency);
-  WVPASSEQ(ntohl(s.tx.acks[ntohl(s.tx.first_ack)].id), 2);
-  WVPASSEQ(ntohl(s.tx.acks[ntohl(s.tx.first_ack)].rxtime), rxtime);
+  WVPASSEQ(ntohl(s.tx.data.acks[s.tx.first_ack].id), 2);
+  WVPASSEQ(ntohl(s.tx.data.acks[s.tx.first_ack].rxtime), rxtime);
   WVPASSEQ(s.num_lost, 0);
   WVPASSEQ(s.lat_tx_count, 1);
   WVPASSEQ(s.lat_tx, half_rtt);
@@ -125,15 +131,15 @@
 
   // Increase the latencies in both directions, reply to client.
   int32_t latency_diff = 10 * 1000;
-  t = send_next_packet(&s, sbase, &c, cbase, sc_latency + latency_diff);
+  t = send_next_ack_packet(&s, sbase, &c, cbase, sc_latency + latency_diff);
 
   rxtime = cbase + t;
   WVPASSEQ(ntohl(s.tx.clockdiff), real_clockdiff + cs_latency);
   WVPASSEQ(c.start_rxtime,
            rxtime - ntohl(s.tx.id) * s.usec_per_pkt - latency_diff);
   WVPASSEQ(c.start_rtxtime, sbase + cs_latency + 10 * 1000 - s.usec_per_pkt);
-  WVPASSEQ(ntohl(c.tx.acks[ntohl(c.tx.first_ack)].id), 2);
-  WVPASSEQ(ntohl(c.tx.acks[ntohl(c.tx.first_ack)].rxtime), rxtime);
+  WVPASSEQ(ntohl(c.tx.data.acks[c.tx.first_ack].id), 2);
+  WVPASSEQ(ntohl(c.tx.data.acks[c.tx.first_ack].rxtime), rxtime);
   WVPASSEQ(c.num_lost, 0);
   WVPASSEQ(c.lat_tx_count, 2);
   WVPASSEQ(c.lat_tx, half_rtt);
@@ -142,15 +148,15 @@
   WVPASSEQ(c.num_lost, 0);
 
   // Client replies with increased latency, server notices.
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency + latency_diff);
+  t = send_next_ack_packet(&c, cbase, &s, sbase, cs_latency + latency_diff);
 
   rxtime = sbase + t;
   WVPASSEQ(ntohl(c.tx.clockdiff), - real_clockdiff + sc_latency);
   WVPASSEQ(s.start_rxtime, sbase + cs_latency - s.usec_per_pkt);
   WVPASSEQ(s.start_rtxtime, cbase - c.usec_per_pkt);
   WVPASSEQ(ntohl(s.rx.clockdiff), cbase - sbase + sc_latency);
-  WVPASSEQ(ntohl(s.tx.acks[ntohl(s.tx.first_ack)].id), 3);
-  WVPASSEQ(ntohl(s.tx.acks[ntohl(s.tx.first_ack)].rxtime), rxtime);
+  WVPASSEQ(ntohl(s.tx.data.acks[s.tx.first_ack].id), 3);
+  WVPASSEQ(ntohl(s.tx.data.acks[s.tx.first_ack].rxtime), rxtime);
   WVPASSEQ(s.num_lost, 0);
   WVPASSEQ(s.lat_tx_count, 2);
   WVPASSEQ(s.lat_tx, half_rtt + latency_diff);
@@ -163,12 +169,12 @@
   s.next_send += s.usec_per_pkt;
   s.next_tx_id++;
 
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency + latency_diff);
+  t = send_next_ack_packet(&c, cbase, &s, sbase, cs_latency + latency_diff);
 
   rxtime = sbase + t;
   WVPASSEQ(ntohl(s.rx.clockdiff), cbase - sbase + sc_latency);
-  WVPASSEQ(ntohl(s.tx.acks[ntohl(s.tx.first_ack)].id), 3);
-  WVPASSEQ(ntohl(s.tx.acks[ntohl(s.tx.first_ack)].rxtime),
+  WVPASSEQ(ntohl(s.tx.data.acks[s.tx.first_ack].id), 3);
+  WVPASSEQ(ntohl(s.tx.data.acks[s.tx.first_ack].rxtime),
            rxtime - s.usec_per_pkt);
   WVPASSEQ(s.num_lost, 0);
   WVPASSEQ(s.lat_tx_count, 2);
@@ -179,11 +185,11 @@
 
   // Remove the extra latency from server->client, send the next packet, have
   // the client receive it and notice the lost packet and reduced latency.
-  t = send_next_packet(&s, sbase, &c, cbase, sc_latency);
+  t = send_next_ack_packet(&s, sbase, &c, cbase, sc_latency);
 
   rxtime = cbase + t;
-  WVPASSEQ(ntohl(c.tx.acks[ntohl(c.tx.first_ack)].id), 4);
-  WVPASSEQ(ntohl(c.tx.acks[ntohl(c.tx.first_ack)].rxtime), rxtime);
+  WVPASSEQ(ntohl(c.tx.data.acks[c.tx.first_ack].id), 4);
+  WVPASSEQ(ntohl(c.tx.data.acks[c.tx.first_ack].rxtime), rxtime);
   WVPASSEQ(c.num_lost, 1);
   WVPASSEQ(c.lat_tx_count, 4);
   WVPASSEQ(c.lat_tx, half_rtt + latency_diff);
@@ -194,7 +200,8 @@
   // A tiny reduction in latency shows up in min_cycle_rxdiff.
   latency_diff = 0;
   int32_t latency_mini_diff = -15;
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency + latency_mini_diff);
+  t = send_next_ack_packet(&c, cbase, &s, sbase,
+                           cs_latency + latency_mini_diff);
 
   WVPASSEQ(ntohl(s.rx.clockdiff), cbase - sbase + sc_latency);
   WVPASSEQ(s.min_cycle_rxdiff, latency_mini_diff);
@@ -202,7 +209,8 @@
   WVPASSEQ(s.lat_tx, half_rtt);
   WVPASSEQ(s.lat_rx, half_rtt + latency_mini_diff);
 
-  t = send_next_packet(&s, sbase, &c, cbase, sc_latency + latency_mini_diff);
+  t = send_next_ack_packet(&s, sbase, &c, cbase,
+                           sc_latency + latency_mini_diff);
 
   WVPASSEQ(ntohl(s.rx.clockdiff), cbase - sbase + sc_latency);
   WVPASSEQ(c.min_cycle_rxdiff, latency_mini_diff);
@@ -212,7 +220,7 @@
   // Reduce the latency dramatically, verify that both sides see it, and the
   // start time is modified (not the min_cycle_rxdiff).
   latency_diff = -22 * 1000;
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency + latency_diff);
+  t = send_next_ack_packet(&c, cbase, &s, sbase, cs_latency + latency_diff);
 
   WVPASSEQ(ntohl(s.rx.clockdiff), cbase - sbase + sc_latency);
   WVPASSEQ(s.min_cycle_rxdiff, latency_mini_diff);
@@ -223,7 +231,7 @@
   WVPASSEQ(s.lat_tx, half_rtt + latency_diff/2 + latency_mini_diff);
   WVPASSEQ(s.lat_rx, half_rtt + latency_diff/2);
 
-  t = send_next_packet(&s, sbase, &c, cbase, sc_latency + latency_diff);
+  t = send_next_ack_packet(&s, sbase, &c, cbase, sc_latency + latency_diff);
 
   // Now we see the new latency applied to both sides.
   WVPASSEQ(ntohl(s.rx.clockdiff), cbase - sbase + sc_latency);
@@ -233,7 +241,7 @@
 
   // Restore latency on one side of the connection, verify that we track it on
   // only one side and we've improved our clock sync.
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency);
+  t = send_next_ack_packet(&c, cbase, &s, sbase, cs_latency);
 
   WVPASSEQ(ntohl(s.rx.clockdiff), cbase - sbase + sc_latency + latency_diff);
   WVPASSEQ(s.lat_tx, half_rtt + latency_diff);
@@ -241,7 +249,7 @@
 
   // And double-check that the other side also sees the improved clock sync and
   // one-sided latency on the correct side.
-  t = send_next_packet(&s, sbase, &c, cbase, sc_latency + latency_diff);
+  t = send_next_ack_packet(&s, sbase, &c, cbase, sc_latency + latency_diff);
 
   WVPASSEQ(ntohl(c.rx.clockdiff), sbase - cbase + cs_latency + latency_diff);
   WVPASSEQ(c.lat_tx, half_rtt);
@@ -252,10 +260,14 @@
 WVTEST_MAIN("isoping clock drift") {
   uint32_t cbase = 1400 * 1000;
   uint32_t sbase = 1600 * 1000;
+  uint32_t usec_per_pkt = 100 * 1000;
 
   // The states of the client and server.
-  struct Session c(cbase);
-  struct Session s(sbase);
+  struct sockaddr_storage empty_sockaddr;
+  struct Session c(cbase, usec_per_pkt, empty_sockaddr, sizeof(empty_sockaddr));
+  struct Session s(sbase, usec_per_pkt, empty_sockaddr, sizeof(empty_sockaddr));
+  c.handshake_state = Session::ESTABLISHED;
+  s.handshake_state = Session::ESTABLISHED;
   // Send packets infrequently, to get new cycles more often.
   s.usec_per_pkt = 1 * 1000 * 1000;
   c.usec_per_pkt = 1 * 1000 * 1000;
@@ -269,7 +281,7 @@
 
   // Perform the initial setup.
   c.next_send = cbase;
-  uint32_t t = send_next_packet(&c, cbase, &s, sbase, cs_latency);
+  uint32_t t = send_next_ack_packet(&c, cbase, &s, sbase, cs_latency);
   s.next_send = sbase + t + 10 * 1000;
 
   uint32_t orig_server_start_rxtime = s.start_rxtime;
@@ -280,7 +292,7 @@
   WVPASSEQ(s.lat_tx, 0);
   WVPASSEQ(s.min_cycle_rxdiff, 0);
 
-  t = send_next_packet(&s, sbase, &c, cbase, sc_latency);
+  t = send_next_ack_packet(&s, sbase, &c, cbase, sc_latency);
 
   uint32_t orig_client_start_rxtime = c.start_rxtime;
   WVPASSEQ(c.start_rxtime, cbase + 2 * half_rtt + 10 * 1000 - c.usec_per_pkt);
@@ -292,7 +304,7 @@
 
   // Clock drift shows up as symmetric changes in one-way latency.
   int32_t total_drift = drift_per_round;
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency + total_drift);
+  t = send_next_ack_packet(&c, cbase, &s, sbase, cs_latency + total_drift);
 
   WVPASSEQ(s.start_rxtime, orig_server_start_rxtime);
   WVPASSEQ(s.start_rtxtime, cbase - c.usec_per_pkt);
@@ -301,7 +313,7 @@
   WVPASSEQ(s.lat_tx, half_rtt);
   WVPASSEQ(s.min_cycle_rxdiff, 0);
 
-  t = send_next_packet(&s, sbase, &c, cbase, sc_latency - total_drift);
+  t = send_next_ack_packet(&s, sbase, &c, cbase, sc_latency - total_drift);
 
   WVPASSEQ(c.start_rxtime, cbase + 2 * half_rtt + 10 * 1000 - c.usec_per_pkt);
   WVPASSEQ(c.start_rtxtime,
@@ -313,7 +325,7 @@
 
   // Once we exceed -20us of drift, we adjust the client's start_rxtime.
   total_drift += drift_per_round;
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency + total_drift);
+  t = send_next_ack_packet(&c, cbase, &s, sbase, cs_latency + total_drift);
 
   WVPASSEQ(s.start_rxtime, orig_server_start_rxtime);
   WVPASSEQ(s.start_rtxtime, cbase - c.usec_per_pkt);
@@ -322,7 +334,7 @@
   WVPASSEQ(s.lat_tx, half_rtt - drift_per_round);
   WVPASSEQ(s.min_cycle_rxdiff, 0);
 
-  t = send_next_packet(&s, sbase, &c, cbase, sc_latency - total_drift);
+  t = send_next_ack_packet(&s, sbase, &c, cbase, sc_latency - total_drift);
 
   int32_t clock_adj = total_drift;
   WVPASSEQ(c.start_rxtime,
@@ -344,7 +356,7 @@
   total_drift += packets_to_skip * drift_per_round;
 
   // At first we blame the rx latency for most of the drift.
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency + total_drift);
+  t = send_next_ack_packet(&c, cbase, &s, sbase, cs_latency + total_drift);
 
   // start_rxtime doesn't change here as the first cycle suppresses positive
   // min_cycle_rxdiff values.
@@ -357,7 +369,7 @@
   WVPASSEQ(s.min_cycle_rxdiff, INT_MAX);
 
   // After one round-trip, we divide the blame for the latency diff evenly.
-  t = send_next_packet(&s, sbase, &c, cbase, sc_latency - total_drift);
+  t = send_next_ack_packet(&s, sbase, &c, cbase, sc_latency - total_drift);
 
   WVPASSEQ(c.start_rxtime, orig_client_start_rxtime - total_drift);
   WVPASSEQ(c.start_rtxtime, sbase + cs_latency + 10 * 1000 - c.usec_per_pkt);
@@ -366,7 +378,7 @@
   WVPASSEQ(c.lat_tx, half_rtt + total_drift / 2);
   WVPASSEQ(c.min_cycle_rxdiff, INT_MAX);
 
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency + total_drift);
+  t = send_next_ack_packet(&c, cbase, &s, sbase, cs_latency + total_drift);
 
   WVPASSEQ(s.start_rxtime, orig_server_start_rxtime);
   WVPASSEQ(s.start_rtxtime, cbase - c.usec_per_pkt);
@@ -377,7 +389,7 @@
   WVPASSEQ(s.min_cycle_rxdiff, total_drift);
 
   total_drift += drift_per_round;
-  t = send_next_packet(&s, sbase, &c, cbase, sc_latency - total_drift);
+  t = send_next_ack_packet(&s, sbase, &c, cbase, sc_latency - total_drift);
   // And on the client.  The client doesn't notice the total_drift rxdiff as it
   // was swallowed by the new cycle.
   WVPASSEQ(c.min_cycle_rxdiff, -drift_per_round);
@@ -392,7 +404,7 @@
   c.next_tx_id += packets_to_skip;
   total_drift += packets_to_skip * drift_per_round;
   int32_t drift_per_cycle = 10 * drift_per_round;
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency + total_drift);
+  t = send_next_ack_packet(&c, cbase, &s, sbase, cs_latency + total_drift);
 
   // The clock drift has worked its way into the RTT calculation.
   half_rtt = (cs_latency + sc_latency - drift_per_cycle) / 2;
@@ -406,7 +418,7 @@
   WVPASSEQ(s.lat_tx, half_rtt - drift_per_round);
   WVPASSEQ(s.min_cycle_rxdiff, INT_MAX);
 
-  t = send_next_packet(&s, sbase, &c, cbase, sc_latency - total_drift);
+  t = send_next_ack_packet(&s, sbase, &c, cbase, sc_latency - total_drift);
 
   WVPASSEQ(c.start_rxtime, orig_client_start_rxtime - total_drift);
   WVPASSEQ(c.start_rtxtime, sbase + cs_latency + 10 * 1000 - c.usec_per_pkt);
@@ -414,7 +426,426 @@
   WVPASSEQ(c.lat_rx, half_rtt + drift_per_round / 2);
   WVPASSEQ(c.lat_tx, half_rtt + total_drift / 2 + 1);
   WVPASSEQ(c.min_cycle_rxdiff, INT_MAX);
+}
 
-  t = send_next_packet(&c, cbase, &s, sbase, cs_latency + total_drift);
+WVTEST_MAIN("Send and receive on sockets") {
+  uint32_t cbase = 1400 * 1000;
+  uint32_t sbase = 1600 * 1000;
 
+  // Sockets for the client and server.
+  int ssock, csock;
+  struct addrinfo hints, *res;
+
+  // Get local interface information.
+  memset(&hints, 0, sizeof(hints));
+  hints.ai_family = AF_INET6;
+  hints.ai_socktype = SOCK_DGRAM;
+  hints.ai_flags = AI_PASSIVE | AI_V4MAPPED;
+  int err = getaddrinfo(NULL, "0", &hints, &res);
+  if (err != 0) {
+    WVPASSEQ("Error from getaddrinfo: ", gai_strerror(err));
+    return;
+  }
+
+  ssock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
+  if (!WVPASS(ssock >= 0)) {
+    perror("server socket");
+    return;
+  }
+
+  csock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
+  if (!WVPASS(csock >= 0)) {
+    perror("client socket");
+    return;
+  }
+
+  if (!WVPASS(!bind(ssock, res->ai_addr, res->ai_addrlen))) {
+    perror("bind");
+    return;
+  }
+
+  // Figure out the local port we got.
+  struct sockaddr_storage listenaddr;
+  socklen_t listenaddr_len = sizeof(listenaddr);
+  memset(&listenaddr, 0, listenaddr_len);
+  if (!WVPASS(!getsockname(ssock, (struct sockaddr *)&listenaddr,
+                           &listenaddr_len))) {
+    perror("getsockname");
+    return;
+  }
+
+  printf("Bound server socket to port=%d\n",
+         listenaddr.ss_family == AF_INET
+             ? ntohs(((struct sockaddr_in *)&listenaddr)->sin_port)
+             : ntohs(((struct sockaddr_in6 *)&listenaddr)->sin6_port));
+
+  // Connect the client's socket.
+  if (!WVPASS(
+          !connect(csock, (struct sockaddr *)&listenaddr, listenaddr_len))) {
+    perror("connect");
+    return;
+  }
+  struct sockaddr_in6 caddr;
+  socklen_t caddr_len = sizeof(caddr);
+  memset(&caddr, 0, caddr_len);
+  if (!WVPASS(!getsockname(csock, (struct sockaddr *)&caddr, &caddr_len))) {
+    perror("getsockname");
+    return;
+  }
+  char buf[128];
+  inet_ntop(AF_INET6, (struct sockaddr *)&caddr, buf, sizeof(buf));
+  printf("Created client connection on %s:%d\n", buf, ntohs(caddr.sin6_port));
+
+  // All active sessions for the client and server.
+  Sessions c;
+  Sessions s;
+  uint32_t usec_per_pkt = 100 * 1000;
+
+  s.MaybeRotateCookieSecrets();
+  // TODO(pmccurdy): Remove +1?
+  c.NewSession(cbase + 1, usec_per_pkt, &listenaddr, listenaddr_len);
+
+  int is_server = 1;
+  int is_client = 0;
+
+  // Send the initial handshake packet.
+  Session &cSession = c.session_map.begin()->second;
+  uint32_t t = cSession.next_send - cbase;
+  WVPASS(!send_waiting_packets(&c, csock, cbase + t, is_client));
+
+  WVPASSEQ(cSession.handshake_retry_count, 0);
+
+  fd_set rfds;
+  FD_ZERO(&rfds);
+  FD_SET(ssock, &rfds);
+  struct timeval tv = {0, 0};
+  int nfds = select(ssock + 1, &rfds, NULL, NULL, &tv);
+  WVPASSEQ(nfds, 1);
+
+  WVPASS(!read_incoming_packet(&s, ssock, sbase, is_server));
+
+  // The server returns its handshake cookie immediately.
+  FD_ZERO(&rfds);
+  FD_SET(csock, &rfds);
+  nfds = select(csock + 1, &rfds, NULL, NULL, &tv);
+  WVPASSEQ(nfds, 1);
+
+  // Eat the packet before the client can see it.
+  Packet p;
+  WVPASSEQ(recv(csock, &p, sizeof(p), 0), 540);
+
+  // The client doesn't send more packets until the handshake timeout expires.
+  t += Session::handshake_timeout_usec - 1;
+  WVPASS(!send_waiting_packets(&c, csock, cbase + t, is_client));
+  FD_ZERO(&rfds);
+  FD_SET(csock, &rfds);
+  nfds = select(csock + 1, &rfds, NULL, NULL, &tv);
+  WVPASSEQ(nfds, 0);
+
+  // Wait for the client to time out and resend the initial handshake packet.
+  t += 1;
+  WVPASS(!send_waiting_packets(&c, csock, cbase + t, is_client));
+
+  FD_ZERO(&rfds);
+  FD_SET(ssock, &rfds);
+  nfds = select(ssock + 1, &rfds, NULL, NULL, &tv);
+  WVPASSEQ(nfds, 1);
+
+  // The server resends its cookie immediately.
+  WVPASS(!read_incoming_packet(&s, ssock, sbase, is_server));
+
+  // The server doesn't store any state for unverified clients
+  WVPASSEQ(s.session_map.size(), 0);
+
+  // Let the client read the cookie, establishing the connection.
+  WVPASS(!read_incoming_packet(&c, csock, cbase + t, is_client));
+  WVPASSEQ(cSession.next_tx_id, 1);
+
+  uint32_t cs_latency = 4000;
+  uint32_t sc_latency = 5000;
+
+  WVPASSEQ(cSession.next_send, cbase + t);
+  t = cSession.next_send - cbase - 1;
+  WVPASS(!send_waiting_packets(&c, csock, cbase + t, is_client));
+
+  // Verify we didn't send a packet before its time.
+  FD_ZERO(&rfds);
+  FD_SET(ssock, &rfds);
+  nfds = select(ssock + 1, &rfds, NULL, NULL, &tv);
+  WVPASSEQ(nfds, 0);
+
+  // Send a packet in each direction.  The server can now verify the client.
+  t += 1;
+  WVPASSEQ(cSession.next_tx_id, 1);
+  WVPASS(!send_waiting_packets(&c, csock, cbase + t, is_client));
+  WVPASSEQ(cSession.next_tx_id, 2);
+
+  FD_ZERO(&rfds);
+  FD_SET(ssock, &rfds);
+  nfds = select(ssock + 1, &rfds, NULL, NULL, &tv);
+  WVPASSEQ(nfds, 1);
+
+  t += cs_latency;
+  WVPASS(!read_incoming_packet(&s, ssock, sbase + t, is_server));
+  WVPASSEQ(s.session_map.size(), 1);
+  WVPASSEQ(s.next_sends.size(), 1);
+  WVPASSEQ(s.next_send_time(), sbase + t + 10 * 1000);
+
+  WVPASSEQ(s.session_map.size(), 1);
+  Session &sSession = s.session_map.begin()->second;
+  WVPASS(sSession.remoteaddr_len > 0);
+  WVPASSEQ(sSession.next_tx_id, 1);
+  WVPASSEQ(ntohl(sSession.rx.id), 1);
+
+  t = s.next_send_time() - sbase;
+  WVPASS(!send_waiting_packets(&s, ssock, sbase + t, is_server));
+  WVPASSEQ(s.next_send_time(), sbase + t + sSession.usec_per_pkt);
+  WVPASSEQ(sSession.next_tx_id, 2);
+
+  t += sc_latency;
+  WVPASS(!read_incoming_packet(&c, csock, cbase + t, is_client));
+  WVPASSEQ(cSession.lat_rx_count, 1);
+
+  // Verify we reject garbage data.
+  memset(&p, 0, sizeof(p));
+  if (!WVPASSEQ(send(csock, &p, sizeof(p), 0), sizeof(p))) {
+    perror("sendto");
+    return;
+  }
+
+  WVPASSEQ(read_incoming_packet(&s, ssock, sbase + t, is_server), EINVAL);
+
+  // Make a new client, who sends more frequently, getting a new source port.
+  Sessions c2;
+  c2.NewSession(cbase, usec_per_pkt/4, &listenaddr, listenaddr_len);
+  int c2sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
+  if (!WVPASS(c2sock > 0)) {
+    perror("client socket 2");
+    return;
+  }
+  if (!WVPASS(
+          !connect(c2sock, (struct sockaddr *)&listenaddr, listenaddr_len))) {
+    perror("connect");
+    return;
+  }
+  struct sockaddr_in6 c2addr;
+  socklen_t c2addr_len = sizeof(c2addr);
+  memset(&c2addr, 0, c2addr_len);
+  if (!WVPASS(!getsockname(c2sock, (struct sockaddr *)&c2addr, &c2addr_len))) {
+    perror("getsockname");
+    return;
+  }
+  inet_ntop(AF_INET6, (struct sockaddr *)&c2addr, buf, sizeof(buf));
+  printf("Created new client connection on %s:%d\n", buf,
+         ntohs(c2addr.sin6_port));
+
+  Session &c2Session = c2.session_map.begin()->second;
+  // Perform the handshake dance so the server knows we're legit.
+  prepare_tx_packet(&c2Session);
+  WVPASS(!send_packet(&c2Session, c2sock, is_client));
+  t = cs_latency;
+  WVPASS(!read_incoming_packet(&s, ssock, sbase+t, is_server));
+  t += sc_latency;
+  WVPASS(!read_incoming_packet(&c2, c2sock, cbase+t, is_client));
+
+  // Now we can send a validated packet to the server.
+  t = c2Session.next_send - cbase;
+  WVPASS(!send_waiting_packets(&c2, c2sock, cbase + t, is_client));
+
+  t += cs_latency;
+
+  // Check that a new client is added to the server's state, and it will be sent
+  // next.
+  WVPASS(!read_incoming_packet(&s, ssock, sbase + t, is_server));
+  WVPASSEQ(s.session_map.size(), 2);
+  WVPASSEQ(s.next_sends.size(), 2);
+  WVPASSEQ(s.next_send_time(), sbase + t + 10 * 1000);
+
+  // Cleanup
+  close(ssock);
+  close(csock);
+  close(c2sock);
+  freeaddrinfo(res);
+}
+
+WVTEST_MAIN("Cookie Validation") {
+  struct addrinfo hints, *res;
+
+  // Get local interface information.
+  memset(&hints, 0, sizeof(hints));
+  hints.ai_family = AF_INET6;
+  hints.ai_socktype = SOCK_DGRAM;
+  hints.ai_flags = AI_PASSIVE | AI_V4MAPPED;
+  int err = getaddrinfo(NULL, "0", &hints, &res);
+  if (err != 0) {
+    WVPASSEQ("Error from getaddrinfo: ", gai_strerror(err));
+    return;
+  }
+
+  // Set up a socket
+  struct sockaddr_storage addr;
+  socklen_t addr_len = sizeof(addr);
+  memset(&addr, 0, addr_len);
+  int sock;
+  sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
+  if (!WVPASS(sock >= 0)) {
+    perror("socket");
+    return;
+  }
+  if (!WVPASS(!getsockname(sock, (struct sockaddr *)&addr, &addr_len))) {
+    perror("getsockname");
+    return;
+  }
+
+  Sessions s;
+  uint32_t epoch = 1;
+  s.RotateCookieSecrets(epoch);
+  Packet p;
+  memset(&p, 0, sizeof(p));
+  WVFAIL(s.CalculateCookie(&p, &addr, addr_len));
+
+  p.packet_type = PACKET_TYPE_HANDSHAKE;
+  p.usec_per_pkt = 100000;
+  WVPASS(s.CalculateCookie(&p, &addr, addr_len));
+
+  // We validate cookies we generate.
+  WVPASS(s.ValidateCookie(&p, &addr, addr_len));
+
+  // Validation fails after changing the IP port or address.
+  sockaddr_storage changed_addr;
+  memcpy(&changed_addr, &addr, addr_len);
+  ((sockaddr_in6 *)&changed_addr)->sin6_port++;
+  WVFAIL(s.ValidateCookie(&p, &changed_addr, addr_len));
+
+  memcpy(&changed_addr, &addr, addr_len);
+  ((sockaddr_in6 *)&changed_addr)->sin6_addr.s6_addr[0]++;
+  WVFAIL(s.ValidateCookie(&p, &changed_addr, addr_len));
+
+  // Validation fails after changing the usec_per_pkt.
+  p.usec_per_pkt++;
+  WVFAIL(s.ValidateCookie(&p, &addr, addr_len));
+  p.usec_per_pkt--;
+
+  // Validation fails after plain modifying the cookie.
+  p.data.handshake.cookie[0]++;
+  WVFAIL(s.ValidateCookie(&p, &changed_addr, addr_len));
+  p.data.handshake.cookie[0]--;
+
+  // Cookies generated with the previous secret still validate.
+  epoch++;
+  s.RotateCookieSecrets(epoch);
+  WVPASS(s.ValidateCookie(&p, &addr, addr_len));
+
+  // But secrets older than that don't validate.
+  epoch++;
+  s.RotateCookieSecrets(epoch);
+  WVFAIL(s.ValidateCookie(&p, &addr, addr_len));
+
+  // Cleanup
+  close(sock);
+  freeaddrinfo(res);
+}
+
+WVTEST_MAIN("Exponential Handshake Backoff") {
+  struct addrinfo hints, *res;
+
+  // Get local interface information.
+  memset(&hints, 0, sizeof(hints));
+  hints.ai_family = AF_INET6;
+  hints.ai_socktype = SOCK_DGRAM;
+  hints.ai_flags = AI_PASSIVE | AI_V4MAPPED;
+  int err = getaddrinfo(NULL, "0", &hints, &res);
+  if (err != 0) {
+    WVPASSEQ("Error from getaddrinfo: ", gai_strerror(err));
+    return;
+  }
+
+  // Set up a socket
+  struct sockaddr_storage addr;
+  socklen_t addr_len = sizeof(addr);
+  memset(&addr, 0, addr_len);
+  int sock;
+  sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
+  if (!WVPASS(sock >= 0)) {
+    perror("socket");
+    return;
+  }
+  if (!WVPASS(!getsockname(sock, (struct sockaddr *)&addr, &addr_len))) {
+    perror("getsockname");
+    return;
+  }
+
+  uint32_t cbase = 400*1000;
+  uint32_t usec_per_pkt = 100 * 1000;
+  Sessions c;
+  c.NewSession(cbase, usec_per_pkt, &addr, addr_len);
+  Session &cSession = c.session_map.begin()->second;
+  WVPASSEQ(cSession.next_send, cbase);
+
+  // Test that we resend handshake packets on an exponential backoff schedule,
+  // up until round 10.
+  int is_client = 0;
+  send_packet(&cSession, sock, is_client);
+  WVPASSEQ(cSession.handshake_state, Session::HANDSHAKE_REQUESTED);
+  WVPASSEQ(cSession.handshake_retry_count, 0);
+  WVPASSEQ(cSession.next_send, cbase + Session::handshake_timeout_usec);
+
+  uint32_t t = cSession.next_send;
+  send_packet(&cSession, sock, is_client);
+  WVPASSEQ(cSession.handshake_retry_count, 1);
+  WVPASSEQ(cSession.next_send, t + 2 * Session::handshake_timeout_usec);
+
+  t = cSession.next_send;
+  send_packet(&cSession, sock, is_client);
+  WVPASSEQ(cSession.handshake_retry_count, 2);
+  WVPASSEQ(cSession.next_send, t + 4 * Session::handshake_timeout_usec);
+
+  t = cSession.next_send;
+  send_packet(&cSession, sock, is_client);
+  WVPASSEQ(cSession.handshake_retry_count, 3);
+  WVPASSEQ(cSession.next_send, t + 8 * Session::handshake_timeout_usec);
+
+  t = cSession.next_send;
+  send_packet(&cSession, sock, is_client);
+  WVPASSEQ(cSession.handshake_retry_count, 4);
+  WVPASSEQ(cSession.next_send, t + 16 * Session::handshake_timeout_usec);
+
+  t = cSession.next_send;
+  send_packet(&cSession, sock, is_client);
+  WVPASSEQ(cSession.handshake_retry_count, 5);
+  WVPASSEQ(cSession.next_send, t + 32 * Session::handshake_timeout_usec);
+
+  t = cSession.next_send;
+  send_packet(&cSession, sock, is_client);
+  WVPASSEQ(cSession.handshake_retry_count, 6);
+  WVPASSEQ(cSession.next_send, t + 64 * Session::handshake_timeout_usec);
+
+  t = cSession.next_send;
+  send_packet(&cSession, sock, is_client);
+  WVPASSEQ(cSession.handshake_retry_count, 7);
+  WVPASSEQ(cSession.next_send, t + 128 * Session::handshake_timeout_usec);
+
+  t = cSession.next_send;
+  send_packet(&cSession, sock, is_client);
+  WVPASSEQ(cSession.handshake_retry_count, 8);
+  WVPASSEQ(cSession.next_send, t + 256 * Session::handshake_timeout_usec);
+
+  t = cSession.next_send;
+  send_packet(&cSession, sock, is_client);
+  WVPASSEQ(cSession.handshake_retry_count, 9);
+  WVPASSEQ(cSession.next_send, t + 512 * Session::handshake_timeout_usec);
+
+  t = cSession.next_send;
+  send_packet(&cSession, sock, is_client);
+  WVPASSEQ(cSession.handshake_retry_count, 10);
+  WVPASSEQ(cSession.next_send, t + 1024 * Session::handshake_timeout_usec);
+
+  t = cSession.next_send;
+  send_packet(&cSession, sock, is_client);
+  WVPASSEQ(cSession.handshake_retry_count, 11);
+  WVPASSEQ(cSession.next_send, t + 1024 * Session::handshake_timeout_usec);
+
+  // Cleanup
+  close(sock);
+  freeaddrinfo(res);
 }
diff --git a/gpio-mailbox/Makefile b/gpio-mailbox/Makefile
index f6cb77f..bd29573 100644
--- a/gpio-mailbox/Makefile
+++ b/gpio-mailbox/Makefile
@@ -21,10 +21,14 @@
 LDFLAGS += $(EXTRA_LDFLAGS)
 
 # enable the platform we're supporting
-ifeq ($(BR2_PACKAGE_BCM_NEXUS),y)
-  CFLAGS += -DBROADCOM
-else ifeq ($(BR2_PACKAGE_MINDSPEED_DRIVERS),y)
+ifeq ($(BR2_PACKAGE_MINDSPEED_DRIVERS),y)
   CFLAGS += -DMINDSPEED
+else ifeq ($(BR2_TARGET_GENERIC_PLATFORM_NAME),gfibertv)
+  CFLAGS += -DBROADCOM
+else ifeq ($(BR2_TARGET_GENERIC_PLATFORM_NAME),gftv200)
+  CFLAGS += -DBROADCOM
+else ifeq ($(BR2_TARGET_GENERIC_PLATFORM_NAME),gftv254)
+  CFLAGS += -DBROADCOM
 else ifeq ($(BR2_TARGET_GENERIC_PLATFORM_NAME),gflt110)
   CFLAGS += -DGFIBER_LT
 else ifeq ($(BR2_TARGET_GENERIC_PLATFORM_NAME),gflt200)
diff --git a/wifi/configs.py b/wifi/configs.py
index e83ee68..23eac7f 100644
--- a/wifi/configs.py
+++ b/wifi/configs.py
@@ -94,6 +94,7 @@
 {require_vht}
 {hidden}
 {ap_isolate}
+{wds}
 {vendor_elements}
 
 ht_capab={ht20}{ht40}{guard_interval}{ht_rxstbc}
@@ -288,6 +289,7 @@
   hidden = 'ignore_broadcast_ssid=1' if opt.hidden_mode else ''
   bridge = 'bridge=%s' % opt.bridge if opt.bridge else ''
   ap_isolate = 'ap_isolate=1' if opt.client_isolation else ''
+  wds = 'wds_sta=1' if opt.wds else ''
   hostapd_conf_parts = [_HOSTCONF_TPL.format(
       interface=interface, band=band, channel=channel, width=width,
       protocols=protocols, hostapd_band=hostapd_band,
@@ -296,7 +298,7 @@
       ht_rxstbc=ht_rxstbc, vht_settings=vht_settings,
       guard_interval=guard_interval, enable_wmm=enable_wmm, hidden=hidden,
       ap_isolate=ap_isolate, auth_algs=auth_algs, bridge=bridge,
-      ssid=utils.sanitize_ssid(opt.ssid),
+      ssid=utils.sanitize_ssid(opt.ssid), wds=wds,
       vendor_elements=get_vendor_elements(opt))]
 
   if opt.encryption != 'NONE':
diff --git a/wifi/configs_test.py b/wifi/configs_test.py
index 68e7b2a..e088298 100755
--- a/wifi/configs_test.py
+++ b/wifi/configs_test.py
@@ -352,6 +352,32 @@
 
 
 
+
+ht_capab=[HT20][RX-STBC1]
+
+"""
+
+_HOSTAPD_CONFIG_WDS = """ctrl_interface=/var/run/hostapd
+interface=wlan0
+
+ssid=TEST_SSID
+utf8_ssid=1
+auth_algs=1
+hw_mode=g
+channel=1
+country_code=US
+ieee80211d=1
+ieee80211h=1
+ieee80211n=1
+
+
+
+
+
+
+wds_sta=1
+
+
 ht_capab=[HT20][RX-STBC1]
 
 """
@@ -376,6 +402,7 @@
 
 
 
+
 ht_capab=[HT20][RX-STBC1]
 
 """
@@ -398,6 +425,7 @@
 
 ignore_broadcast_ssid=1
 
+
 vendor_elements=dd04f4f5e801dd0df4f5e803544553545f53534944
 
 ht_capab=[HT20][RX-STBC1]
@@ -435,6 +463,7 @@
     self.client_isolation = False
     self.supports_provisioning = False
     self.no_band_restriction = False
+    self.wds = False
 
 
 def wpa_passphrase(ssid, passphrase):
@@ -505,6 +534,17 @@
                   config)
   opt.bridge = default_bridge
 
+  # Test WDS.
+  default_wds, opt.wds = opt.wds, True
+  config = configs.generate_hostapd_config(
+      _PHY_INFO, 'wlan0', '2.4', '1', '20', set(('a', 'b', 'g', 'n', 'ac')),
+      'asdfqwer', opt)
+  wvtest.WVPASSEQ('\n'.join((_HOSTAPD_CONFIG_WDS,
+                             _HOSTAPD_CONFIG_WPA,
+                             '# Experiments: ()\n')),
+                  config)
+  opt.wds = default_wds
+
   # Test provisioning IEs.
   default_hidden_mode, opt.hidden_mode = opt.hidden_mode, True
   default_supports_provisioning, opt.supports_provisioning = (
diff --git a/wifi/iw.py b/wifi/iw.py
index f16c50b..0110267 100644
--- a/wifi/iw.py
+++ b/wifi/iw.py
@@ -421,3 +421,11 @@
 def scan(interface, scan_args):
   """Return 'iw scan' output for printing."""
   return _scan(interface, scan_args)
+
+
+def set_4address_mode(interface, on):
+  try:
+    setting = 'on' if on else 'off'
+    subprocess.check_output(['iw', 'dev', interface, 'set', '4addr', setting])
+  except subprocess.CalledProcessError as e:
+    raise utils.BinWifiException('Failed to set 4addr mode %s: %s', setting, e)
diff --git a/wifi/utils.py b/wifi/utils.py
index 3b2db84..33f3e53 100644
--- a/wifi/utils.py
+++ b/wifi/utils.py
@@ -196,19 +196,20 @@
     return ''
 
 
-def validate_set_wifi_options(band, width, autotype, protocols, encryption):
+def validate_set_wifi_options(opt):
   """Validates options to set_wifi.
 
   Args:
-    band: The specified band, as a string; '2.4' or '5'.
-    width: The specified channel width.
-    autotype: The specified autotype.
-    protocols: The specified 802.11 levels, as a collection of strings.
-    encryption: The specified encryption type.
+    opt: The options to validate.
 
   Raises:
     BinWifiException: if anything is not valid.
   """
+  band = opt.band
+  width = opt.width
+  autotype = opt.autotype
+  protocols = set(opt.protocols.split('/'))
+
   if band not in ('2.4', '5'):
     raise BinWifiException('You must specify band with -b2.4 or -b5')
 
@@ -240,11 +241,14 @@
   elif width == '80' and 'ac' not in protocols:
     raise BinWifiException('-p ac is needed for 40 MHz channels')
 
-  if encryption == 'WEP' or '_PSK_' in encryption:
+  if opt.encryption == 'WEP' or '_PSK_' in opt.encryption:
     if 'WIFI_PSK' not in os.environ:
       raise BinWifiException(
           'Encryption enabled; use WIFI_PSK=whatever wifi set ...')
 
+  if opt.wds and not opt.bridge:
+    raise BinWifiException('WDS mode enabled; must specify a bridge.')
+
 
 def sanitize_ssid(ssid):
   """Remove control and non-UTF8 characters from an SSID.
diff --git a/wifi/utils_test.py b/wifi/utils_test.py
index 8febea3..24e8716 100755
--- a/wifi/utils_test.py
+++ b/wifi/utils_test.py
@@ -2,13 +2,13 @@
 
 """Tests for utils.py."""
 
-import collections
 import multiprocessing
 import os
 import shutil
 import sys
 import tempfile
 
+import configs_test
 import utils
 from wvtest import wvtest
 
@@ -18,8 +18,9 @@
     {'band': '5'},
     {'width': '40'},
     {'autotype': 'ANY'},
-    {'protocols': ('a', 'b', 'ac')},
+    {'protocols': 'a/b/ac'},
     {'encryption': 'NONE'},
+    {'wds': True, 'bridge': 'br0'},
 )
 
 _VALIDATION_FAIL = (
@@ -32,25 +33,23 @@
     {'band': '2.4', 'autotype': 'DFS'},
     {'band': '5', 'autotype': 'OVERLAP'},
     # Invalid protocols
-    {'protocols': set('abc')},
-    {'protocols': set()},
+    {'protocols': 'a/b/c'},
+    {'protocols': ''},
     # Invalid width
     {'width': '25'},
     # Invalid width/protocols
-    {'width': '40', 'protocols': set('abg')},
-    {'width': '80', 'protocols': set('abgn')},
+    {'width': '40', 'protocols': 'a/b/g'},
+    {'width': '80', 'protocols': 'a/b/g/n'},
+    {'wds': True, 'bridge': ''},
 )
 
 
-_DEFAULTS = collections.OrderedDict((('band', '2.4'), ('width', '20'),
-                                     ('autotype', 'NONDFS'),
-                                     ('protocols', ('a', 'b', 'g', 'n', 'ac')),
-                                     ('encryption', 'WPA2_PSK_AES')))
-
-
-def modify_defaults(**kwargs):
-  result = collections.OrderedDict(_DEFAULTS)
-  result.update(kwargs)
+def make_optdict(**kwargs):
+  result = configs_test.FakeOptDict()
+  # This is the default band for 'wifi set'.
+  result.band = '2.4'
+  for k, v in kwargs.iteritems():
+    setattr(result, k, v)
   return result
 
 
@@ -62,23 +61,24 @@
 
   for case in _VALIDATION_PASS:
     try:
-      utils.validate_set_wifi_options(*modify_defaults(**case).values())
+      utils.validate_set_wifi_options(make_optdict(**case))
+      wvtest.WVPASS(True)  # Make WvTest count this as a test.
     except utils.BinWifiException:
       wvtest.WVFAIL('Test failed.')
 
   for case in _VALIDATION_FAIL:
     wvtest.WVEXCEPT(
         utils.BinWifiException, utils.validate_set_wifi_options,
-        *modify_defaults(**case).values())
+        make_optdict(**case))
 
   # Test failure when WIFI_PSK is missing
   del os.environ['WIFI_PSK']
   wvtest.WVEXCEPT(
       utils.BinWifiException, utils.validate_set_wifi_options,
-      *_DEFAULTS.values())
+      make_optdict(**_VALIDATION_PASS[0]))
   wvtest.WVEXCEPT(
       utils.BinWifiException, utils.validate_set_wifi_options,
-      *modify_defaults(encryption='WEP').values())
+      make_optdict(encryption='WEP'))
 
 
 @wvtest.wvtest
diff --git a/wifi/wifi.py b/wifi/wifi.py
index dc00a42..149a243 100755
--- a/wifi/wifi.py
+++ b/wifi/wifi.py
@@ -27,7 +27,7 @@
 
 _OPTSPEC_FORMAT = """
 {bin} set           Enable or modify access points.  Takes all options unless otherwise specified.
-{bin} setclient     Enable or modify wifi clients.  Takes -b, -P, -s, --bssid, -S.
+{bin} setclient     Enable or modify wifi clients.  Takes -b, -P, -s, --bssid, -S, --wds.
 {bin} stop|off      Disable access points and clients.  Takes -b, -P, -S.
 {bin} stopap        Disable access points.  Takes -b, -P, -S.
 {bin} stopclient    Disable wifi clients.  Takes -b, -P, -S.
@@ -53,6 +53,7 @@
 Y,yottasecond-timeouts            Don't rotate any keys: PTK, GTK, or GMK
 P,persist                         For set commands, persist options so we can restore them with 'wifi restore'.  For stop commands, remove persisted options.
 S,interface-suffix=               Interface suffix (defaults to ALL for stop commands; use NONE to specify no suffix) []
+W,wds                             Enable WDS mode (nl80211 only)
 lock-timeout=                     How long, in seconds, to wait for another /bin/wifi process to finish before giving up. [60]
 scan-ap-force                     (Scan only) scan when in AP mode
 scan-passive                      (Scan only) do not probe, scan passively
@@ -228,8 +229,7 @@
   autotype = opt.autotype
   protocols = set(opt.protocols.split('/'))
 
-  utils.validate_set_wifi_options(
-      band, width, autotype, protocols, opt.encryption)
+  utils.validate_set_wifi_options(opt)
 
   psk = None
   if opt.encryption == 'WEP' or '_PSK_' in opt.encryption:
@@ -852,6 +852,10 @@
   if not _stop_hostapd(interface):
     raise utils.BinWifiException("Couldn't stop hostapd")
 
+  # Set or unset 4-address mode.  This has to be done while hostapd is down.
+  utils.log('%s 4-address mode', 'Enabling' if opt.wds else 'Disabling')
+  iw.set_4address_mode(interface, opt.wds)
+
   # We don't want to try to rewrite this file if this is just a forced restart.
   if not forced:
     utils.atomic_write(tmp_config_filename, config)