NAS-140250 / 26.0.0-BETA.1 / Fix keepalived boot deadlock in configure_addresses_impl (by bmeagherix) (#18440)

ix-netif.service runs Before=network-pre.target, but keepalived requires
After=network-online.target. Starting keepalived from
configure_addresses_impl (called via ix-netif.service) caused systemd to
queue the start job for ~95s until network-online.target was eventually
satisfied after ix-netif.service completed - a structural deadlock.

Fix by guarding the keepalived START behind the ix-netif completion
sentinel. If keepalived is already running, RELOAD as before. If it is
not running and the sentinel exists (i.e. we are in a post-boot
interface.sync call), START it. If the sentinel does not exist we are in
the early boot call and skip keepalived entirely; it will be started
once the network is online.

Move NETIF_COMPLETE_SENTINEL from smb_/constants.py to the more
appropriate middlewared/utils/interface.py and update importers
accordingly.

Original PR: #18437

Co-authored-by: Brian M <brian.meagher@ixsystems.com>

Author: bugclerk

src/middlewared/middlewared/plugins/interface/addresses.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import ipaddress
+import os
 import socket
 
 from truenas_pynetif.address.address import add_address, remove_address, replace_address
@@ -11,6 +12,7 @@
 
 from middlewared.plugins.interface.dhcp import dhcp_leases, dhcp_status, dhcp_stop
 from middlewared.service import ServiceContext
+from middlewared.utils.interface import NETIF_COMPLETE_SENTINEL
 
 from .sync_data import SyncData
 
@@ -170,17 +172,6 @@ def configure_addresses_impl(
         "tunable.set_sysctl", f"net.ipv6.conf.{name}.autoconf", autoconf
     )
 
-    # Handle keepalived for VIPs
-    if vip or alias_vips:
-        if not ctx.middleware.call_sync("service.started", "keepalived"):
-            ctx.middleware.call_sync(
-                "service.control", "START", "keepalived"
-            ).wait_sync(raise_error=True)
-        else:
-            ctx.middleware.call_sync(
-                "service.control", "RELOAD", "keepalived"
-            ).wait_sync(raise_error=True)
-
     # Add addresses in database but not configured
     for addr in addrs_database - addrs_configured:
         address = addr.address
@@ -195,6 +186,27 @@ def configure_addresses_impl(
             broadcast=addr.broadcast,
         )
 
+    # Bring interface up
+    if not (link.flags & IFFlags.UP):
+        set_link_up(sock, index=link_index)
+
+    # Handle keepalived for VIPs.  Skip the START during early boot (when called
+    # from ix-netif.service) because keepalived requires network-online.target
+    # which cannot be reached until ix-netif.service itself completes — starting
+    # it here would deadlock for 95s.  The sentinel is written by ix-netif.service
+    # on successful completion, so its presence means the network is up.
+    if vip or alias_vips:
+        if ctx.middleware.call_sync("service.started", "keepalived"):
+            ctx.middleware.call_sync(
+                "service.control", "RELOAD", "keepalived"
+            ).wait_sync(raise_error=True)
+        elif os.path.exists(NETIF_COMPLETE_SENTINEL):
+            ctx.middleware.call_sync(
+                "service.control", "START", "keepalived"
+            ).wait_sync(raise_error=True)
+        # else: early boot call from ix-netif.service; keepalived will be
+        # started later once network-online.target is satisfied.
+
     # Configure MTU (skip for bond members)
     skip_mtu = sync_data.is_bond_member(name)
     if not skip_mtu:
@@ -213,9 +225,5 @@ def configure_addresses_impl(
                 "Failed to set interface description on %s", name, exc_info=True
             )
 
-    # Bring interface up
-    if not (link.flags & IFFlags.UP):
-        set_link_up(sock, index=link_index)
-
     # Return True if DHCP should be started
     return not status.running and data["int_dhcp"]

src/middlewared/middlewared/plugins/smb.py

@@ -25,12 +25,12 @@
 from middlewared.service import ConfigService, ValidationError, ValidationErrors
 from middlewared.service_exception import CallError, MatchNotFound
 from middlewared.plugins.smb_.constants import (
-    NETIF_COMPLETE_SENTINEL,
     CONFIGURED_SENTINEL,
     SMB_AUDIT_DEFAULTS,
     SMBCmd,
     SMBPath,
 )
+from middlewared.utils.interface import NETIF_COMPLETE_SENTINEL
 from middlewared.plugins.smb_.constants import VEEAM_REPO_BLOCKSIZE
 from middlewared.plugins.smb_.constants import SMBShareField as share_field
 from middlewared.plugins.smb_.sharesec import remove_share_acl

src/middlewared/middlewared/plugins/smb_/constants.py

@@ -1,11 +1,9 @@
 import enum
 import os
 from middlewared.utils import ctdb
-from middlewared.utils import MIDDLEWARE_RUN_DIR
 from middlewared.utils.directoryservices.krb5_constants import SAMBA_KEYTAB_DIR
 
 
-NETIF_COMPLETE_SENTINEL = f"{MIDDLEWARE_RUN_DIR}/ix-netif-complete"
 CONFIGURED_SENTINEL = '/var/run/samba/.configured'
 SMB_AUDIT_DEFAULTS = {'enable': False, 'watch_list': [], 'ignore_list': []}
 VEEAM_REPO_BLOCKSIZE = 131072

src/middlewared/middlewared/utils/interface.py

@@ -2,8 +2,11 @@
 import os
 import time
 
+from middlewared.utils import MIDDLEWARE_RUN_DIR
+
 
 IFACE_LINK_STATE_MAX_WAIT: int = 60
+NETIF_COMPLETE_SENTINEL = f"{MIDDLEWARE_RUN_DIR}/ix-netif-complete"
 RTF_GATEWAY: int = 0x0002
 RTF_UP: int = 0x0001