Add error handling for un-notified carrier downs when sending packets.

If a packet send failed because the carrier went down without a
netlink notification, then assume the hardware carrier was lost while
the machine was suspended (eg, ethernet cable pulled during suspend).

Simulate a netlink carrier down event and freeze the dhcp state
machine until a netlink carrier up event is received.

The ARP code is not yet handling this issue everywhere, but the
window of opportunity for it to happen there is much shorter.

src/arp.c

@@ -45,6 +45,7 @@
 #include "options.h"
 #include "leasefile.h"
 #include "sockd.h"
+#include "netlink.h"
 
 #define ARP_MSG_SIZE 0x2a
 #define ARP_RETRANS_DELAY 5000 // ms
@@ -246,6 +247,7 @@ static void arp_reopen_fd(struct client_state_t cs[static 1])
 static int arp_send(struct client_state_t cs[static 1],
                     struct arpMsg arp[static 1])
 {
+    int ret = -1;
     struct sockaddr_ll addr = {
         .sll_family = AF_PACKET,
         .sll_ifindex = client_config.ifindex,
@@ -256,27 +258,27 @@ static int arp_send(struct client_state_t cs[static 1],
     if (cs->arpFd < 0) {
         log_warning("%s: arp: Send attempted when no ARP fd is open.",
                     client_config.interface);
-        return -1;
+        return ret;
     }
 
-    ssize_t r;
     if (!check_carrier(cs->arpFd)) {
         log_error("%s: (%s) carrier down; sendto would fail",
                   client_config.interface, __func__);
+        ret = -99;
         goto carrier_down;
     }
-    r = safe_sendto(cs->arpFd, (const char *)arp, sizeof *arp, 0,
+    ret = safe_sendto(cs->arpFd, (const char *)arp, sizeof *arp, 0,
                     (struct sockaddr *)&addr, sizeof addr);
-    if (r < 0 || (size_t)r != sizeof *arp) {
+    if (ret < 0 || (size_t)ret != sizeof *arp) {
-        if (r < 0)
+        if (ret < 0)
             log_error("%s: (%s) sendto failed: %s",
                       client_config.interface, __func__, strerror(errno));
         else
-            log_error("%s: (%s) sendto short write: %z < %zu",
+            log_error("%s: (%s) sendto short write: %d < %zu",
-                      client_config.interface, __func__, r, sizeof *arp);
+                      client_config.interface, __func__, ret, sizeof *arp);
 carrier_down:
         arp_reopen_fd(cs);
-        return -1;
+        return ret;
     }
     return 0;
 }
@@ -299,8 +301,9 @@ static int arp_ping(struct client_state_t cs[static 1], uint32_t test_ip)
     BASE_ARPMSG();
     memcpy(arp.sip4, &cs->clientAddr, sizeof cs->clientAddr);
     memcpy(arp.dip4, &test_ip, sizeof test_ip);
-    if (arp_send(cs, &arp) < 0)
+    int r = arp_send(cs, &arp);
-        return -1;
+    if (r < 0)
+        return r;
     garp.send_stats[ASEND_GW_PING].count++;
     garp.send_stats[ASEND_GW_PING].ts = curms();
     return 0;
@@ -314,8 +317,9 @@ static int arp_ip_anon_ping(struct client_state_t cs[static 1],
     memcpy(arp.dip4, &test_ip, sizeof test_ip);
     log_line("%s: arp: Probing for hosts that may conflict with our lease...",
              client_config.interface);
-    if (arp_send(cs, &arp) < 0)
+    int r = arp_send(cs, &arp);
-        return -1;
+    if (r < 0)
+        return r;
     garp.send_stats[ASEND_COLLISION_CHECK].count++;
     garp.send_stats[ASEND_COLLISION_CHECK].ts = curms();
     return 0;
@@ -326,8 +330,9 @@ static int arp_announcement(struct client_state_t cs[static 1])
     BASE_ARPMSG();
     memcpy(arp.sip4, &cs->clientAddr, 4);
     memcpy(arp.dip4, &cs->clientAddr, 4);
-    if (arp_send(cs, &arp) < 0)
+    int r = arp_send(cs, &arp);
-        return -1;
+    if (r < 0)
+        return r;
     garp.send_stats[ASEND_ANNOUNCE].count++;
     garp.send_stats[ASEND_ANNOUNCE].ts = curms();
     return 0;
@@ -404,7 +409,18 @@ static void arp_failed(struct client_state_t cs[static 1])
 {
     log_line("%s: arp: Offered address is in use.  Declining.",
              client_config.interface);
-    send_decline(cs, garp.dhcp_packet.yiaddr);
+    int r = send_decline(cs, garp.dhcp_packet.yiaddr);
+    if (r < 0) {
+        log_warning("%s: Failed to send a decline notice packet.",
+                    client_config.interface);
+        if (r == -99) {
+            // Carrier went down while suspended.
+            cs->ifsPrevState = IFS_DOWN;
+            ifnocarrier_action(cs);
+            garp.wake_ts[AS_COLLISION_CHECK] = -1;
+            return;
+        }
+    }
     garp.wake_ts[AS_COLLISION_CHECK] = -1;
     reinit_selecting(cs, garp.total_conflicts < MAX_CONFLICTS ?
                      0 : RATE_LIMIT_INTERVAL);

src/dhcp.c

@@ -111,6 +111,7 @@ static ssize_t send_dhcp_unicast(struct client_state_t cs[static 1],
     if (!check_carrier(fd)) {
         log_error("%s: (%s) carrier down; write would fail",
                   client_config.interface, __func__);
+        ret = -99;
         goto out_fd;
     }
     ret = safe_write(fd, (const char *)payload, payload_len);
@@ -304,6 +305,7 @@ static ssize_t send_dhcp_raw(struct dhcpmsg payload[static 1])
     if (!check_carrier(fd)) {
         log_error("%s: (%s) carrier down; sendto would fail",
                   client_config.interface, __func__);
+        ret = -99;
         goto carrier_down;
     }
     ret = safe_sendto(fd, (const char *)&iudmsg, iud_len, 0,

src/state.c

@@ -40,6 +40,16 @@
 #include "options.h"
 #include "ndhc.h"
 #include "sys.h"
+#include "netlink.h"
+
+// Simulates netlink carrier down event if carrier went down while suspended.
+#define SUSPEND_IF_NOCARRIER() \
+    if (r == -99) { \
+        cs->ifsPrevState = IFS_DOWN; \
+        ifnocarrier_action(cs); \
+        dhcp_wake_ts = -1; \
+        return; \
+    }
 
 static void selecting_packet(struct client_state_t cs[static 1],
                              struct dhcpmsg packet[static 1],
@@ -133,14 +143,18 @@ static void set_released(struct client_state_t cs[static 1])
 static void requesting_timeout(struct client_state_t cs[static 1],
                                long long nowts)
 {
-    if (num_dhcp_requests < 5) {
+    if (num_dhcp_requests >= 5) {
-        if (send_selecting(cs) < 0)
+        reinit_selecting(cs, 0);
+        return;
+    }
+    int r = send_selecting(cs);
+    if (r < 0) {
         log_warning("%s: Failed to send a selecting request packet.",
                     client_config.interface);
+        SUSPEND_IF_NOCARRIER();
+    }
     dhcp_wake_ts = nowts + delay_timeout(cs, num_dhcp_requests);
     num_dhcp_requests++;
-    } else
-        reinit_selecting(cs, 0);
 }
 
 // Triggered when the lease has been held for a significant fraction of its
@@ -167,19 +181,22 @@ static void renewing_timeout(struct client_state_t cs[static 1],
                              long long nowts)
 {
     long long rbt = cs->leaseStartTime + cs->rebindTime * 1000;
-    if (nowts < rbt) {
+    if (nowts >= rbt) {
+        cs->dhcpState = DS_REBINDING;
+        rebinding_timeout(cs, nowts);
+        return;
+    }
     if (rbt - nowts < 30000) {
         dhcp_wake_ts = rbt;
         return;
     }
-        if (send_renew(cs) < 0)
+    int r = send_renew(cs);
+    if (r < 0) {
         log_warning("%s: Failed to send a renew request packet.",
                     client_config.interface);
-        dhcp_wake_ts = nowts + ((rbt - nowts) / 2);
+        SUSPEND_IF_NOCARRIER();
-    } else {
-        cs->dhcpState = DS_REBINDING;
-        rebinding_timeout(cs, nowts);
     }
+    dhcp_wake_ts = nowts + ((rbt - nowts) / 2);
 }
 
 // Triggered when a DHCP rebind request has been sent and no reply has been
@@ -190,20 +207,23 @@ static void rebinding_timeout(struct client_state_t cs[static 1],
                               long long nowts)
 {
     long long elt = cs->leaseStartTime + cs->lease * 1000;
-    if (nowts < elt) {
+    if (nowts >= elt) {
+        log_line("%s: Lease expired.  Searching for a new lease...",
+                 client_config.interface);
+        reinit_selecting(cs, 0);
+        return;
+    }
     if (elt - nowts < 30000) {
         dhcp_wake_ts = elt;
         return;
     }
-        if (send_rebind(cs) < 0)
+    int r = send_rebind(cs);
+    if (r < 0) {
         log_warning("%s: Failed to send a rebind request packet.",
                     client_config.interface);
-        dhcp_wake_ts = nowts + ((elt - nowts) / 2);
+        SUSPEND_IF_NOCARRIER();
-    } else {
-        log_line("%s: Lease expired.  Searching for a new lease...",
-                 client_config.interface);
-        reinit_selecting(cs, 0);
     }
+    dhcp_wake_ts = nowts + ((elt - nowts) / 2);
 }
 
 static void released_timeout(struct client_state_t cs[static 1],
@@ -346,9 +366,12 @@ static void selecting_timeout(struct client_state_t cs[static 1],
     }
     if (num_dhcp_requests == 0)
         cs->xid = nk_random_u32(&cs->rnd32_state);
-    if (send_discover(cs) < 0)
+    int r = send_discover(cs);
+    if (r < 0) {
         log_warning("%s: Failed to send a discover request packet.",
                     client_config.interface);
+        SUSPEND_IF_NOCARRIER();
+    }
     dhcp_wake_ts = nowts + delay_timeout(cs, num_dhcp_requests);
     num_dhcp_requests++;
 }
@@ -363,9 +386,12 @@ static void xmit_release(struct client_state_t cs[static 1])
               svrbuf, sizeof svrbuf);
     log_line("%s: Unicasting a release of %s to %s.", client_config.interface,
              clibuf, svrbuf);
-    if (send_release(cs) < 0)
+    int r = send_release(cs);
+    if (r < 0) {
         log_warning("%s: Failed to send a release request packet.",
                     client_config.interface);
+        SUSPEND_IF_NOCARRIER();
+    }
     print_release(cs);
 }
 
@@ -382,9 +408,12 @@ static void frenew(struct client_state_t cs[static 1])
         log_line("%s: Forcing a DHCP renew...", client_config.interface);
         cs->dhcpState = DS_RENEWING;
         start_dhcp_listen(cs);
-        if (send_renew(cs) < 0)
+        int r = send_renew(cs);
+        if (r < 0) {
             log_warning("%s: Failed to send a renew request packet.",
                         client_config.interface);
+            SUSPEND_IF_NOCARRIER();
+        }
     } else if (cs->dhcpState == DS_RELEASED)
         reinit_selecting(cs, 0);
 }