- An optimization described in the failover protocol draft is now included,

which permits a DHCP server operating in communications-interrupted state
  to 'rewind' a lease to the state most recently transmitted to its peer,
  greatly increasing a server's endurance in communications-interrupted.
  This is supported using a new 'rewind state' record on the dhcpd.leases
  entry for each lease.  [ISC-Bugs #19601]

RELNOTES

@@ -52,6 +52,13 @@ work on other platforms. Please report any problems and suggested fixes to
 
 - Cleaned up some compiler warnings
 
+- An optimization described in the failover protocol draft is now included,
+  which permits a DHCP server operating in communications-interrupted state
+  to 'rewind' a lease to the state most recently transmitted to its peer,
+  greatly increasing a server's endurance in communications-interrupted.
+  This is supported using a new 'rewind state' record on the dhcpd.leases
+  entry for each lease.
+
 			Changes since 4.1.0 (new features)
 
 - Failover port configuration can now be left to defaults (port 647) as

common/conflex.c

@@ -1229,55 +1229,64 @@ intern(char *atom, enum dhcp_token dfv) {
 			return PAUSED;
 		break;
 	      case 'r':
-		if (!strcasecmp (atom + 1, "esolution-interrupted"))
-			return RESOLUTION_INTERRUPTED;
 		if (!strcasecmp(atom + 1, "ange"))
 			return RANGE;
-		if (!strcasecmp(atom + 1, "ange6")) {
+		if (!strcasecmp(atom + 1, "ange6"))
 			return RANGE6;
-		}
+		if (isascii(atom[1]) && (tolower(atom[1]) == 'e')) {
-		if (!strcasecmp (atom + 1, "ecover"))
+			if (!strcasecmp(atom + 2, "bind"))
-			return RECOVER;
-		if (!strcasecmp (atom + 1, "ecover-done"))
-			return RECOVER_DONE;
-		if (!strcasecmp (atom + 1, "ecover-wait"))
-			return RECOVER_WAIT;
-		if (!strcasecmp (atom + 1, "econtact-interval"))
-			return RECONTACT_INTERVAL;
-		if (!strcasecmp (atom + 1, "equest"))
-			return REQUEST;
-		if (!strcasecmp (atom + 1, "equire"))
-			return REQUIRE;
-		if (!strcasecmp (atom + 1, "equire"))
-			return REQUIRE;
-		if (!strcasecmp (atom + 1, "etry"))
-			return RETRY;
-		if (!strcasecmp (atom + 1, "eturn"))
-			return RETURN;
-		if (!strcasecmp (atom + 1, "enew"))
-			return RENEW;
-		if (!strcasecmp (atom + 1, "ebind"))
 				return REBIND;
-		if (!strcasecmp (atom + 1, "eboot"))
+			if (!strcasecmp(atom + 2, "boot"))
 				return REBOOT;
-		if (!strcasecmp (atom + 1, "eject"))
+			if (!strcasecmp(atom + 2, "contact-interval"))
-			return REJECT;
+				return RECONTACT_INTERVAL;
-		if (!strcasecmp (atom + 1, "everse"))
+			if (!strncasecmp(atom + 2, "cover", 5)) {
-			return REVERSE;
+				if (atom[7] == '\0')
-		if (!strcasecmp (atom + 1, "elease"))
+					return RECOVER;
-			return RELEASE;
+				if (!strcasecmp(atom + 7, "-done"))
-		if (!strcasecmp (atom + 1, "efused"))
+					return RECOVER_DONE;
-			return NS_REFUSED;
+				if (!strcasecmp(atom + 7, "-wait"))
-		if (!strcasecmp (atom + 1, "eleased"))
+					return RECOVER_WAIT;
-			return TOKEN_RELEASED;
+				break;
-		if (!strcasecmp (atom + 1, "eset"))
+			}
-			return TOKEN_RESET;
+			if (!strcasecmp(atom + 2, "fresh"))
-		if (!strcasecmp (atom + 1, "eserved"))
-			return TOKEN_RESERVED;
-		if (!strcasecmp (atom + 1, "emove"))
-			return REMOVE;
-		if (!strcasecmp (atom + 1, "efresh"))
 				return REFRESH;
+			if (!strcasecmp(atom + 2, "fused"))
+				return NS_REFUSED;
+			if (!strcasecmp(atom + 2, "ject"))
+				return REJECT;
+			if (!strcasecmp(atom + 2, "lease"))
+				return RELEASE;
+			if (!strcasecmp(atom + 2, "leased"))
+				return TOKEN_RELEASED;
+			if (!strcasecmp(atom + 2, "move"))
+				return REMOVE;
+			if (!strcasecmp(atom + 2, "new"))
+				return RENEW;
+			if (!strcasecmp(atom + 2, "quest"))
+				return REQUEST;
+			if (!strcasecmp(atom + 2, "quire"))
+				return REQUIRE;
+			if (isascii(atom[2]) && (tolower(atom[2]) == 's')) {
+				if (!strcasecmp(atom + 3, "erved"))
+					return TOKEN_RESERVED;
+				if (!strcasecmp(atom + 3, "et"))
+					return TOKEN_RESET;
+				if (!strcasecmp(atom + 3,
+						"olution-interrupted"))
+					return RESOLUTION_INTERRUPTED;
+				break;
+			}
+			if (!strcasecmp(atom + 2, "try"))
+				return RETRY;
+			if (!strcasecmp(atom + 2, "turn"))
+				return RETURN;
+			if (!strcasecmp(atom + 2, "verse"))
+				return REVERSE;
+			if (!strcasecmp(atom + 2, "wind"))
+				return REWIND;
+			break;
+		}
 		break;
 	      case 's':
 		if (!strcasecmp(atom + 1, "cript"))

includes/dhcpd.h

@@ -495,13 +495,28 @@ struct lease {
 					 RESERVED_LEASE | \
 					 BOOTP_LEASE)
 
+	/*
+	 * The lease's binding state is its current state.  The next binding
+	 * state is the next state this lease will move into by expiration,
+	 * or timers in general.  The desired binding state is used on lease
+	 * updates; the caller is attempting to move the lease to the desired
+	 * binding state (and this may either succeed or fail, so the binding
+	 * state must be preserved).
+	 *
+	 * The 'rewind' binding state is used in failover processing.  It
+	 * is used for an optimization when out of communications; it allows
+	 * the server to "rewind" a lease to the previous state acknowledged
+	 * by the peer, and progress forward from that point.
+	 */
 	binding_state_t binding_state;
 	binding_state_t next_binding_state;
 	binding_state_t desired_binding_state;
+	binding_state_t rewind_binding_state;
 
 	struct lease_state *state;
 
-	/* 'tsfp' is more of an 'effective' tsfp.  It may be calculated from
+	/*
+	 * 'tsfp' is more of an 'effective' tsfp.  It may be calculated from
 	 * stos+mclt for example if it's an expired lease and the server is
 	 * in partner-down state.  'atsfp' is zeroed whenever a lease is
 	 * updated - and only set when the peer acknowledges it.  This

includes/dhctoken.h

@@ -356,7 +356,8 @@ enum dhcp_token {
 	ANYCAST_MAC = 659,
 	CONFLICT_DONE = 660,
 	AUTO_PARTNER_DOWN = 661,
-	GETHOSTNAME = 662
+	GETHOSTNAME = 662,
+	REWIND = 663
 };
 
 #define is_identifier(x)	((x) >= FIRST_TOKEN &&	\

server/confpars.c

@@ -3095,6 +3095,16 @@ int parse_lease_declaration (struct lease **lp, struct parse *cfile)
 			}
 			goto do_binding_state;
 
+		      case REWIND:
+			seenbit = 512;
+			token = next_token(&val, NULL, cfile);
+			if (token != BINDING) {
+				parse_warn(cfile, "expecting 'binding'");
+				skip_to_semi(cfile);
+				break;
+			}
+			goto do_binding_state;
+
 		      case BINDING:
 			seenbit = 256;
 
@@ -3152,12 +3162,25 @@ int parse_lease_declaration (struct lease **lp, struct parse *cfile)
 			if (seenbit == 256) {
 				lease -> binding_state = new_state;
 
-				/* If no next binding state is specified, it's
+				/*
-				   the same as the current state. */
+				 * Apply default/conservative next/rewind
+				 * binding states if they haven't been set
+				 * yet.  These defaults will be over-ridden if
+				 * they are set later in parsing.
+				 */
 				if (!(seenmask & 128))
 				    lease->next_binding_state = new_state;
-			} else
+
+				/* The most conservative rewind state. */
+				if (!(seenmask & 512))
+				    lease->rewind_binding_state = new_state;
+			} else if (seenbit == 128)
 				lease -> next_binding_state = new_state;
+			else if (seenbit == 512)
+				lease->rewind_binding_state = new_state;
+			else
+				log_fatal("Impossible condition at %s:%d.",
+					  MDL);
 
 			parse_semi (cfile);
 			break;
@@ -3406,6 +3429,9 @@ int parse_lease_declaration (struct lease **lp, struct parse *cfile)
 				lease -> next_binding_state = FTS_FREE;
 		} else
 			lease -> next_binding_state = lease -> binding_state;
+
+		/* The most conservative rewind state implies no rewind. */
+		lease->rewind_binding_state = lease->binding_state;
 	}
 
 	if (!(seenmask & 65536))

server/db.c

@@ -168,6 +168,20 @@ int write_lease (lease)
 			  : "abandoned")) < 0)
                         ++errors;
 
+	/*
+	 * In this case, if the rewind state is not present in the lease file,
+	 * the reader will use the current binding state as the most
+	 * conservative (safest) state.  So if the in-memory rewind state is
+	 * for some reason invalid, the best thing to do is not to write a
+	 * state and let the reader take on a safe state.
+	 */
+	if ((lease->binding_state != lease->rewind_binding_state) &&
+	    (lease->rewind_binding_state > 0) &&
+	    (lease->rewind_binding_state <= FTS_LAST) &&
+	    (fprintf(db_file, "\n  rewind binding state %s;",
+		     binding_state_names[lease->rewind_binding_state-1])) < 0)
+			++errors;
+
 	if (lease->flags & RESERVED_LEASE)
 		if (fprintf(db_file, "\n  reserved;") < 0)
                         ++errors;

server/dhcp.c

@@ -312,12 +312,19 @@ void dhcpdiscover (packet, ms_nulltp)
 	if (lease && lease -> pool && lease -> pool -> failover_peer) {
 		peer = lease -> pool -> failover_peer;
 
-		/* If the lease is ours to allocate, then allocate it.
+		/*
+		 * If the lease is ours to (re)allocate, then allocate it.
+		 *
 		 * If the lease is active, it belongs to the client.  This
 		 * is the right lease, if we are to offer one.  We decide
 		 * whether or not to offer later on.
+		 *
+		 * If the lease was last active, and we've reached this
+		 * point, then it was last active with the same client.  We
+		 * can safely re-activate the lease with this client.
 		 */
 		if (lease->binding_state == FTS_ACTIVE ||
+		    lease->rewind_binding_state == FTS_ACTIVE ||
 		    lease_mine_to_reallocate(lease)) {
 			; /* This space intentionally left blank. */
 
@@ -521,13 +528,18 @@ void dhcprequest (packet, ms_nulltp, ip_lease)
 			goto out;
 		}
 
-		/* If the lease is in a transitional state, we can't
+		/*
-		   renew it. */
+		 * If the lease is in a transitional state, we can't
+		 * renew it unless we can rewind it to a non-transitional
+		 * state (active, free, or backup).  lease_mine_to_reallocate()
+		 * checks for free/backup, so we only need to check for active.
+		 */
 		if ((lease->binding_state == FTS_RELEASED ||
 		     lease->binding_state == FTS_EXPIRED) &&
+		    lease->rewind_binding_state != FTS_ACTIVE &&
 		    !lease_mine_to_reallocate(lease)) {
 			log_debug("%s: lease in transition state %s", msgbuf,
-				   lease -> binding_state == FTS_RELEASED
+				  (lease->binding_state == FTS_RELEASED)
 				   ? "released" : "expired");
 			goto out;
 		}
@@ -3315,7 +3327,8 @@ int find_lease (struct lease **lp,
 		goto out;
 	}
 
-	/* If we found leases matching the client identifier, loop through
+	/*
+	 * If we found leases matching the client identifier, loop through
 	 * the n_uid pointer looking for one that's actually valid.   We
 	 * can't do this until we get here because we depend on
 	 * packet -> known, which may be set by either the uid host
@@ -3331,13 +3344,19 @@ int find_lease (struct lease **lp,
 #endif
 
 #if defined (FAILOVER_PROTOCOL)
-		/* When failover is active, it's possible that there could
+		/*
-		   be two "free" leases for the same uid, but only one of
+		 * When we lookup a lease by uid, we know the client identifier
-		   them that's available for this failover peer to allocate. */
+		 * matches the lease's record.  If it is active, or was last
+		 * active with the same client, we can trivially extend it.
+		 * If is not or was not active, we can allocate it to this
+		 * client if it matches the usual free/backup criteria (which
+		 * is contained in lease_mine_to_reallocate()).
+		 */
 		if (uid_lease->binding_state != FTS_ACTIVE &&
+		    uid_lease->rewind_binding_state != FTS_ACTIVE &&
 		    !lease_mine_to_reallocate(uid_lease)) {
 #if defined (DEBUG_FIND_LEASE)
-			log_info ("not mine to allocate: %s",
+			log_info("not active or not mine to allocate: %s",
 				 piaddr(uid_lease->ip_addr));
 #endif
 			goto n_uid;
@@ -3398,19 +3417,32 @@ int find_lease (struct lease **lp,
 			  piaddr (hw_lease -> ip_addr));
 #endif
 #if defined (FAILOVER_PROTOCOL)
-		/* When failover is active, it's possible that there could
+		/*
-		   be two "free" leases for the same uid, but only one of
+		 * When we lookup a lease by chaddr, we know the MAC address
-		   them that's available for this failover peer to allocate. */
+		 * matches the lease record (we will check if the lease has a
+		 * client-id the client does not next).  If the lease is
+		 * currently active or was last active with this client, we can
+		 * trivially extend it.  Otherwise, there are a set of rules
+		 * that govern if we can reallocate this lease to any client
+		 * ("lease_mine_to_reallocate()") including this one.
+		 */
 		if (hw_lease->binding_state != FTS_ACTIVE &&
+		    hw_lease->rewind_binding_state != FTS_ACTIVE &&
 		    !lease_mine_to_reallocate(hw_lease)) {
 #if defined (DEBUG_FIND_LEASE)
-			log_info ("not mine to allocate: %s",
+			log_info("not active or not mine to allocate: %s",
 				 piaddr(hw_lease->ip_addr));
 #endif
 			goto n_hw;
 		}
 #endif
 
+		/*
+		 * This conditional skips "potentially active" leases (leases
+		 * we think are expired may be extended by the peer, etc) that
+		 * may be assigned to a differently /client-identified/ client
+		 * with the same MAC address.
+		 */
 		if (hw_lease -> binding_state != FTS_FREE &&
 		    hw_lease -> binding_state != FTS_BACKUP &&
 		    hw_lease -> uid &&
@@ -3495,8 +3527,15 @@ int find_lease (struct lease **lp,
 		lease_dereference (&ip_lease, MDL);
 	}
 
-	/* Toss ip_lease if it hasn't yet expired and doesn't belong to the
+	/*
-	   client. */
+	 * If the requested address is in use (or potentially in use) by
+	 * a different client, it can't be granted.
+	 *
+	 * This first conditional only detects if the lease is currently
+	 * identified to a different client (client-id and/or chaddr
+	 * mismatch).  In this case we may not want to give the client the
+	 * lease, if doing so may potentially be an addressing conflict.
+	 */
 	if (ip_lease &&
 	    (ip_lease -> uid ?
 	     (!have_client_identifier ||
@@ -3508,11 +3547,14 @@ int find_lease (struct lease **lp,
 	      memcmp (&ip_lease -> hardware_addr.hbuf [1],
 		      packet -> raw -> chaddr,
 		      (unsigned)(ip_lease -> hardware_addr.hlen - 1))))) {
-		/* If we're not doing failover, the only state in which
+		/*
-		   we can allocate this lease to the client is FTS_FREE.
+		 * A lease is unavailable for allocation to a new client if
-		   If we are doing failover, things are more complicated.
+		 * it is not in the FREE or BACKUP state.  There may be
-		   If the lease is free or backup, we let the caller decide
+		 * leases that are in the expired state with a rewinding
-		   whether or not to give it out. */
+		 * state that is free or backup, but these will be processed
+		 * into the free or backup states by expiration processes, so
+		 * checking for them here is superfluous.
+		 */
 		if (ip_lease -> binding_state != FTS_FREE &&
 		    ip_lease -> binding_state != FTS_BACKUP) {
 #if defined (DEBUG_FIND_LEASE)
@@ -3526,16 +3568,19 @@ int find_lease (struct lease **lp,
 		}
 	}
 
-	/* If we got an ip_lease and a uid_lease or hw_lease, and ip_lease
+	/*
-	   is not active, and is not ours to reallocate, forget about it. */
+	 * If we got an ip_lease and a uid_lease or hw_lease, and ip_lease
+	 * is/was not active, and is not ours to reallocate, forget about it.
+	 */
 	if (ip_lease && (uid_lease || hw_lease) &&
 	    ip_lease->binding_state != FTS_ACTIVE &&
+	    ip_lease->rewind_binding_state != FTS_ACTIVE &&
 #if defined(FAILOVER_PROTOCOL)
 	    !lease_mine_to_reallocate(ip_lease) &&
 #endif
 	    packet->packet_type == DHCPDISCOVER) {
 #if defined (DEBUG_FIND_LEASE)
-		log_info ("ip lease not ours to offer.");
+		log_info("ip lease not active or not ours to offer.");
 #endif
 		lease_dereference(&ip_lease, MDL);
 	}
@@ -3973,6 +4018,7 @@ int allocate_lease (struct lease **lp, struct packet *packet,
 				}
 			}
 
+			/* Try abandoned leases as a last resort. */
 			if ((candl == NULL) &&
 			    (pool->abandoned != NULL) &&
 			    lease_mine_to_reallocate(pool->abandoned))
@@ -4003,6 +4049,18 @@ int allocate_lease (struct lease **lp, struct packet *packet,
 			continue;
 		}
 
+		/*
+		 * There are tiers of lease state preference, listed here in
+		 * reverse order (least to most preferential):
+		 *
+		 *    ABANDONED
+		 *    FREE/BACKUP
+		 *
+		 * If the selected lease and candidate are both of the same
+		 * state, select the oldest (longest ago) expiration time
+		 * between the two.  If the candidate lease is of a higher
+		 * preferred grade over the selected lease, use it.
+		 */
 		if ((lease -> binding_state == FTS_ABANDONED) &&
 		    ((candl -> binding_state != FTS_ABANDONED) ||
 		     (candl -> ends < lease -> ends))) {

server/failover.c

@@ -4584,6 +4584,36 @@ isc_result_t dhcp_failover_send_bind_update (dhcp_failover_state_t *state,
 
 	lease->last_xid = link->xid++;
 
+	/*
+	 * Our very next action is to transmit a binding update relating to
+	 * this lease over the wire, and although there is a BNDACK, there is
+	 * no BNDACKACK or BNDACKACKACK...the basic issue as we send a BNDUPD,
+	 * we may not receive a BNDACK.  This non-reception does not imply the
+	 * peer did not receive and process the BNDUPD.  So at this point, we
+	 * must divest any state that would be dangerous to retain under the
+	 * impression the peer has been updated.  Normally state changes like
+	 * this are processed in supersede_lease(), but in this case we need a
+	 * very late binding.
+	 *
+	 * In failover rules, a server is permitted to work forward in certain
+	 * directions from a given lease's state; active leases may be
+	 * extended, so forth.  There is an 'optimization' in the failover
+	 * draft that permits a server to 'rewind' any work they have not
+	 * informed the peer.  Since we can't know if the peer received our
+	 * update but was unable to acknowledge it, we make this change on
+	 * transmit rather than upon receiving the acknowledgement.
+	 *
+	 * XXX: Frequent lease commits are undesirable.  This should hopefully
+	 * only trigger when a server is sending a lease /state change/, and
+	 * not merely an update such as with a renewal.
+	 */
+	if (lease->rewind_binding_state != lease->binding_state) {
+		lease->rewind_binding_state = lease->binding_state;
+
+		write_lease(lease);
+		commit_leases();
+	}
+
 	/* Send the update. */
 	status = (dhcp_failover_put_message
 		  (link, link -> outer,
@@ -5296,6 +5326,12 @@ isc_result_t dhcp_failover_process_bind_update (dhcp_failover_state_t *state,
 	}
 	msg -> binding_status = lt -> next_binding_state;
 
+	/*
+	 * If we accept a peer's binding update, then we can't rewind a
+	 * lease behind the peer's state.
+	 */
+	lease->rewind_binding_state = lt->next_binding_state;
+
 	/* Try to install the new information. */
 	if (!supersede_lease (lease, lt, 0, 0, 0) ||
 	    !write_lease (lease)) {
@@ -5435,8 +5471,13 @@ isc_result_t dhcp_failover_process_bind_ack (dhcp_failover_state_t *state,
 			lease->next_binding_state = FTS_BACKUP;
 		else
 			lease->next_binding_state = FTS_FREE;
+
 		/* Clear this condition for the next go-round. */
 		lease->desired_binding_state = lease->next_binding_state;
+
+		/* The peer will have made this state change, so set rewind. */
+		lease->rewind_binding_state = lease->next_binding_state;
+
 		supersede_lease(lease, (struct lease *)0, 0, 0, 0);
 		write_lease(lease);
 
@@ -6158,6 +6199,12 @@ int lease_mine_to_reallocate (struct lease *lease)
 
 	if (lease && lease->pool &&
 	    (peer = lease->pool->failover_peer)) {
+		/*
+		 * In addition to the normal rules governing wether a server
+		 * is allowed to operate changes on a lease, the server is
+		 * allowed to operate on a lease from the standpoint of the
+		 * most conservative guess of the peer's state for this lease.
+		 */
 		switch (lease->binding_state) {
 		      case FTS_ACTIVE:
 			/* ACTIVE leases may not be reallocated. */
@@ -6182,23 +6229,46 @@ int lease_mine_to_reallocate (struct lease *lease)
 				(peer->me.stos + peer->mclt < cur_time) :
 				(lease->tsfp + peer->mclt < cur_time)));
 
-		      case FTS_RESET:
 		      case FTS_RELEASED:
 		      case FTS_EXPIRED:
-			/* These three lease states go onto the 'expired'
+			/*
-			 * queue.  Upon entry into partner-down state, this
+			 * These leases are generally untouchable until the
-			 * queue of leases has their tsfp values modified
+			 * peer acknowledges their state change.  However, as
-			 * to equal stos+mclt, the point at which the server
+			 * this is impossible if the peer is offline, the
-			 * is allowed to remove them from these transitional
+			 * failover protocol permits an 'optimization' to
-			 * states without an acknowledgement.
+			 * rewind the lease to a previous state that the server
+			 * is allowed to operate on, if that was the state that
+			 * was last acknowledged by the peer.
+			 *
+			 * So if a lease was free, was allocated by this
+			 * server, and expired without ever being transmitted
+			 * to the peer, it can be returned to free and given
+			 * to any new client legally.
+			 */
+			if ((peer->i_am == primary) &&
+			    (lease->rewind_binding_state == FTS_FREE))
+				return 1;
+			if ((peer->i_am == secondary) &&
+			    (lease->rewind_binding_state == FTS_BACKUP))
+				return 1;
+
+			/* FALL THROUGH (released, expired, reset) */
+		      case FTS_RESET:
+			/*
+			 * Released, expired, and reset leases go onto the
+			 * 'expired' queue all together.  Upon entry into
+			 * partner-down state, this queue of leases has their
+			 * tsfp values modified to equal stos+mclt, the point
+			 * at which the server is allowed to remove them from
+			 * these transitional states.
 			 *
 			 * Note that although tsfp has been possibly extended
 			 * past the actual tsfp we received from the peer, we
 			 * don't have to take any special action.  Since tsfp
-			 * is now in the past (or now), we can guarantee that
+			 * will be equal to the current time when the lease
-			 * this server will only allocate a lease time equal
+			 * transitions to free, tsfp will not be used to grant
-			 * to MCLT, rather than a TSFP-optimal lease, which is
+			 * lease-times longer than the MCLT to clients, which
-			 * the only danger for a lease in one of these states.
+			 * is the only danger for this sort of modification.
 			 */
 			return((peer->service_state == service_partner_down) &&
 			       (lease->tsfp < cur_time));

server/mdb.c

@@ -801,14 +801,14 @@ void new_address_range (cfile, low, high, subnet, pool, lpchain)
 						    i + min)),
 				   isc_result_totext (status));
 #endif
-		lp -> ip_addr = ip_addr (subnet -> net,
+		lp->ip_addr = ip_addr(subnet->net, subnet->netmask, i + min);
-					 subnet -> netmask, i + min);
 		lp->starts = MIN_TIME;
 		lp->ends = MIN_TIME;
 		subnet_reference(&lp->subnet, subnet, MDL);
 		pool_reference(&lp->pool, pool, MDL);
 		lp->binding_state = FTS_FREE;
 		lp->next_binding_state = FTS_FREE;
+		lp->rewind_binding_state = FTS_FREE;
 		lp->flags = 0;
 
 		/* Remember the lease in the IP address hash. */
@@ -1224,7 +1224,8 @@ int supersede_lease (comp, lease, commit, propogate, pimmediate)
 
       just_move_it:
 #if defined (FAILOVER_PROTOCOL)
-	/* Atsfp should be cleared upon any state change that implies
+	/*
+	 * Atsfp should be cleared upon any state change that implies
 	 * propagation whether supersede_lease was given a copy lease
 	 * structure or not (often from the pool_timer()).
 	 */
@@ -1356,6 +1357,24 @@ int supersede_lease (comp, lease, commit, propogate, pimmediate)
 	}
 
 	if (commit) {
+#if defined(FAILOVER_PROTOCOL)
+		/*
+		 * If commit and propogate are set, then we can save a
+		 * possible fsync later in BNDUPD socket transmission by
+		 * stepping the rewind state forward to the new state, in
+		 * case it has changed.  This is only worth doing if the
+		 * failover connection is currently connected, as in this
+		 * case it is likely we will be transmitting to the peer very
+		 * shortly.
+		 */
+		if (propogate && (comp->pool->failover_peer != NULL) &&
+		    ((comp->pool->failover_peer->service_state ==
+							    cooperating) ||
+		     (comp->pool->failover_peer->service_state ==
+							    not_responding)))
+			comp->rewind_binding_state = comp->binding_state;
+#endif
+
 		if (!write_lease (comp))
 			return 0;
 		if ((server_starting & SS_NOSYNC) == 0) {
@@ -1405,8 +1424,7 @@ void make_binding_state_transition (struct lease *lease)
 #if defined (FAILOVER_PROTOCOL)
 		    peer &&
 		    (lease->binding_state == FTS_EXPIRED ||
-		     (peer -> i_am == secondary &&
+		     lease->binding_state == FTS_ACTIVE) &&
-		      lease -> binding_state == FTS_ACTIVE)) &&
 		    (lease->next_binding_state == FTS_FREE ||
 		     lease->next_binding_state == FTS_BACKUP)) ||
 	     (!peer &&
@@ -1564,7 +1582,6 @@ void make_binding_state_transition (struct lease *lease)
 		   piaddr (lease -> ip_addr),
 		   binding_state_print (lease -> next_binding_state));
 #endif
-
 }
 
 /* Copy the contents of one lease into another, correctly maintaining
@@ -1636,6 +1653,7 @@ int lease_copy (struct lease **lp,
 	lt->cltt = lease -> cltt;
 	lt->binding_state = lease->binding_state;
 	lt->next_binding_state = lease->next_binding_state;
+	lt->rewind_binding_state = lease->rewind_binding_state;
 	status = lease_reference(lp, lt, file, line);
 	lease_dereference(&lt, MDL);
 	return status == ISC_R_SUCCESS;
@@ -1690,6 +1708,19 @@ void release_lease (lease, packet)
 		lease->tstp = cur_time;
 #if defined (FAILOVER_PROTOCOL)
 		if (lease -> pool && lease -> pool -> failover_peer) {
+			dhcp_failover_state_t *peer = NULL;
+
+			if (lease->pool != NULL)
+				peer = lease->pool->failover_peer;
+
+			if ((peer->service_state == not_cooperating) &&
+			    (((peer->i_am == primary) &&
+			      (lease->rewind_binding_state == FTS_FREE)) ||
+			     ((peer->i_am == secondary) &&
+			      (lease->rewind_binding_state == FTS_BACKUP)))) {
+				lease->next_binding_state =
+						  lease->rewind_binding_state;
+			} else
 				lease -> next_binding_state = FTS_RELEASED;
 		} else {
 			lease -> next_binding_state = FTS_FREE;
@@ -1803,11 +1834,24 @@ void pool_timer (vpool)
 #if defined (FAILOVER_PROTOCOL)
 		if (pool->failover_peer &&
 		    pool->failover_peer->me.state != partner_down) {
-			/* The secondary can't remove a lease from the
+			/*
-			   active state except in partner_down. */
+			 * Normally the secondary doesn't initiate expiration
-			if (i == ACTIVE_LEASES &&
+			 * events (unless in partner-down), but rather relies
-			    pool -> failover_peer -> i_am == secondary)
+			 * on the primary to expire the lease.  However, when
+			 * disconnected from its peer, the server is allowed to
+			 * rewind a lease to the previous state that the peer
+			 * would have recorded it.  This means there may be
+			 * opportunities for active->free or active->backup
+			 * expirations while out of contact.
+			 *
+			 * Q: Should we limit this expiration to
+			 *    comms-interrupt rather than not-normal?
+			 */
+			if ((i == ACTIVE_LEASES) &&
+			    (pool->failover_peer->i_am == secondary) &&
+			    (pool->failover_peer->me.state == normal))
 				continue;
+
 			/* Leases in an expired state don't move to
 			   free because of a timeout unless we're in
 			   partner_down. */
@@ -1837,10 +1881,29 @@ void pool_timer (vpool)
 			   state change should happen, just call
 			   supersede_lease on it to make the change
 			   happen. */
-			if (lease -> next_binding_state !=
+			if (lease->next_binding_state != lease->binding_state)
-			    lease -> binding_state)
+			{
-				supersede_lease (lease,
+#if defined(FAILOVER_PROTOCOL)
-						 (struct lease *)0, 1, 1, 1);
+				dhcp_failover_state_t *peer = NULL;
+
+				if (lease->pool != NULL)
+					peer = lease->pool->failover_peer;
+
+				/* Can we rewind the lease to a free state? */
+				if (peer != NULL &&
+				    peer->service_state == not_cooperating &&
+				    lease->next_binding_state == FTS_EXPIRED &&
+				    ((peer->i_am == primary &&
+				      lease->rewind_binding_state == FTS_FREE)
+					||
+				     (peer->i_am == secondary &&
+				      lease->rewind_binding_state ==
+								FTS_BACKUP)))
+					lease->next_binding_state =
+						   lease->rewind_binding_state;
+#endif
+				supersede_lease(lease, NULL, 1, 1, 1);
+			}
 
 			lease_dereference (&lease, MDL);
 			if (next)
@@ -2287,9 +2350,8 @@ int write_leases ()
 		for (i = FREE_LEASES; i <= RESERVED_LEASES; i++) {
 		    for (l = *(lptr [i]); l; l = l -> next) {
 #if !defined (DEBUG_DUMP_ALL_LEASES)
-			if (l -> hardware_addr.hlen ||
+			if (l->hardware_addr.hlen != 0 || l->uid_len != 0 ||
-			    l -> uid_len ||
+			    l->tsfp != 0 || l->binding_state != FTS_FREE)
-			    (l -> binding_state != FTS_FREE))
 #endif
 			{
 			    if (!write_lease (l))