mountpoint_cmp: flatten mountpoint= paramater before sorting

Datasets with nested mountpoints must be mounted in order from
shallowest to deepest, otherwise the deeper ones will be obscured by the
shallow ones.

mountpoint_cmp() has only traditionally been a character-based sort,
with no awareness of "legal" patterns in pathnames, in particular, runs
of '/' characters. These are (for better or worse) legal paths, but
confuse a character-based comparison, causing mounts to be attempted in
an incorrect order, with mixed results.

This commit adds a path "flattening" function, and applies it to the
string returned from the mountpoint= paramater before using it as a
path. This corrects the sort order and allows mounting to proceed
correctly.

Reported-by: James McKay <portablejim@jamesmckay.id.au>
Sponsored-by: TrueNAS
Signed-off-by: Rob Norris <rob.norris@truenas.com>

Author: robn

include/libzutil.h

@@ -126,6 +126,7 @@ _LIBZUTIL_H char *zfs_strip_partition(const char *);
 _LIBZUTIL_H const char *zfs_strip_path(const char *);
 
 _LIBZUTIL_H int zfs_strcmp_pathname(const char *, const char *, int);
+_LIBZUTIL_H int zfs_flatten_path(char *, size_t, char **);
 
 _LIBZUTIL_H boolean_t zfs_dev_is_dm(const char *);
 _LIBZUTIL_H boolean_t zfs_dev_is_whole_disk(const char *);

lib/libzfs/libzfs.abi

@@ -379,6 +379,7 @@
     <elf-symbol name='zfs_device_get_physical' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_dirnamelen' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_expand_proplist' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='zfs_flatten_path' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_foreach_mountpoint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_get_all_props' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_get_clones_nvl' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -5957,6 +5958,12 @@
       <parameter type-id='80f4b756'/>
       <return type-id='48b5725f'/>
     </function-decl>
+    <function-decl name='zfs_flatten_path' mangled-name='zfs_flatten_path' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_flatten_path'>
+      <parameter type-id='26a90f95'/>
+      <parameter type-id='b59d7dce'/>
+      <parameter type-id='9b23c9ad'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='mkdirp' mangled-name='mkdirp' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='mkdirp'>
       <parameter type-id='80f4b756'/>
       <parameter type-id='d50d396c'/>
@@ -6554,6 +6561,11 @@
     <pointer-type-def type-id='35acf840' size-in-bits='64' id='17f3480d'/>
     <pointer-type-def type-id='688c495b' size-in-bits='64' id='cec6f2e4'/>
     <pointer-type-def type-id='d11b7617' size-in-bits='64' id='23432aaa'/>
+    <function-decl name='dump_nvlist' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='5ce45b60'/>
+      <parameter type-id='95e97e5e'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-decl name='zpool_get_handle' mangled-name='zpool_get_handle' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_handle'>
       <parameter type-id='4c81de99'/>
       <return type-id='b0382bb3'/>

lib/libzfs/libzfs_mount.c

@@ -28,6 +28,7 @@
  * Copyright 2017 RackTop Systems.
  * Copyright (c) 2018 Datto Inc.
  * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
+ * Copyright (c) 2026, TrueNAS.
  */
 
 /*
@@ -269,6 +270,7 @@ zfs_is_mountable(zfs_handle_t *zhp, char *buf, size_t buflen,
 	if (source)
 		*source = sourcetype;
 
+	VERIFY0(zfs_flatten_path(buf, buflen, NULL));
 	return (B_TRUE);
 }
 
@@ -998,11 +1000,13 @@ mountpoint_cmp(const void *arga, const void *argb)
 	if (gota) {
 		verify(zfs_prop_get(za, ZFS_PROP_MOUNTPOINT, mounta,
 		    sizeof (mounta), NULL, NULL, 0, B_FALSE) == 0);
+		VERIFY0(zfs_flatten_path(mounta, sizeof (mounta), NULL));
 	}
 	gotb = (zfs_get_type(zb) == ZFS_TYPE_FILESYSTEM);
 	if (gotb) {
 		verify(zfs_prop_get(zb, ZFS_PROP_MOUNTPOINT, mountb,
 		    sizeof (mountb), NULL, NULL, 0, B_FALSE) == 0);
+		VERIFY0(zfs_flatten_path(mountb, sizeof (mountb), NULL));
 	}
 
 	if (gota && gotb) {
@@ -1061,10 +1065,13 @@ non_descendant_idx(zfs_handle_t **handles, size_t num_handles, int idx)
 
 	verify(zfs_prop_get(handles[idx], ZFS_PROP_MOUNTPOINT, parent,
 	    sizeof (parent), NULL, NULL, 0, B_FALSE) == 0);
+	VERIFY0(zfs_flatten_path(parent, sizeof (parent), NULL));
 
 	for (i = idx + 1; i < num_handles; i++) {
 		verify(zfs_prop_get(handles[i], ZFS_PROP_MOUNTPOINT, child,
 		    sizeof (child), NULL, NULL, 0, B_FALSE) == 0);
+		VERIFY0(zfs_flatten_path(child, sizeof (child), NULL));
+
 		if (!libzfs_path_contains(parent, child))
 			break;
 	}
@@ -1170,6 +1177,7 @@ zfs_mount_task(void *arg)
 
 	verify(zfs_prop_get(handles[idx], ZFS_PROP_MOUNTPOINT, mountpoint,
 	    sizeof (mountpoint), NULL, NULL, 0, B_FALSE) == 0);
+	VERIFY0(zfs_flatten_path(mountpoint, sizeof (mountpoint), NULL));
 
 	if (mp->mnt_func(handles[idx], mp->mnt_data) != 0)
 		goto out;
@@ -1187,6 +1195,7 @@ zfs_mount_task(void *arg)
 		char child[ZFS_MAXPROPLEN];
 		verify(zfs_prop_get(handles[i], ZFS_PROP_MOUNTPOINT,
 		    child, sizeof (child), NULL, NULL, 0, B_FALSE) == 0);
+		VERIFY0(zfs_flatten_path(child, sizeof (child), NULL));
 
 		if (!libzfs_path_contains(mountpoint, child))
 			break; /* not a descendant, return */

lib/libzutil/zutil_device_path.c

@@ -22,6 +22,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026, Rob Norris <robn@despairlabs.com>
  */
 
 #include <errno.h>
@@ -203,3 +204,76 @@ zfs_strcmp_pathname(const char *name, const char *cmp, int wholedisk)
 
 	return (0);
 }
+
+/*
+ * "Flatten" a UNIX-style path by:
+ * - collapsing multiple '/' chars to just one
+ * - removing trailing '/' chars (one or more followed by '\0')
+ *
+ * args:
+ * - src: source buffer
+ * - srclen: size of source buffer (>= strlen(in)+1)
+ * - dstp: pointer to destination buffer:
+ *   - if NULL, source buffer is modified in place
+ *   - if set and *dstp is set, output written to *dstp
+ *   - if set and *dstp is NULL, *dstp set to malloc(srclen)
+ *
+ * returns:
+ * - 0 on success. caller must free *dstp is it was allocated (above)
+ * - ENOMEM: alloc failed
+ */
+int
+zfs_flatten_path(char *src, size_t srclen, char **dstp)
+{
+	char *s, *d, *dstbuf = NULL;
+
+	s = src;
+	if (dstp == NULL)
+		/* Modify in-place */
+		d = src;
+	else if (*dstp != NULL)
+		/* Use provided dest buffer */
+		d = *dstp;
+	else {
+		/* Allocate dest buffer */
+		dstbuf = malloc(srclen);
+		if (dstbuf == NULL)
+			return (ENOMEM);
+		d = dstbuf;
+	}
+
+	while (*s != '\0') {
+		/* Copy char unless in-place and no change */
+		if (s != d)
+			*d = *s;
+
+		/* Found a slash */
+		if (*s == '/') {
+			/* Move past additional slashes */
+			while (*s == '/')
+				s++;
+
+			/*
+			 * Found null after slash, abort. d still points
+			 * at the slash copied above, which will then be
+			 * nulled after the loop.
+			 */
+			if (*s == '\0')
+				break;
+		} else
+			/* Non-slash char copied, move source forward. */
+			s++;
+
+		/* Copied char ok, move dest forward for next iteration */
+		d++;
+	}
+
+	/* End of path, set dest terminator unless in-place and no move. */
+	if (s != d)
+		*d = '\0';
+
+	if (dstp != NULL && *dstp == NULL)
+		*dstp = dstbuf;
+
+	return (0);
+}