linkfile: use fops->create in dht_linkfile_create

A newly created linkfile will be deleted for the race, after
this, the gfid between linkfile and datafile may get
mismatched. And shards deletion process is also affected
by such a mismatch, most shards in BRICK_PATH/.shard
cannot be removed correctly and space cannot be released.

To fix this race, replace mknod with create during linkfile
creation, in this case, we can keep this linkfile opening
during the whole linkfile and datafile creations.

A linkfile will not be regarded as stale and deleted then,
because its fd is opened and posix_unlink can handle it
properly.

In this patch, the linkfile and datafile creation process is as below:

    dht                       hashed_subvol                cached_subvol
--> |                              |                              |
    |                              |                              |
    | a.create tmp fd as local->fd |                              |
    | b.send fops->create          |                              |
    |---- dht_linkfile_create ---->|                              |
    |                              |                              |
    | c.succeed to create linkfile,|                              |
    |<-- dht_linkfile_create_cbk --|                              |
    |                                                             |
    |--- corresponding fop for datafile, mknod, create, etc.  --->|
    |                                                             |
    |<----------            datafile fop cbk            ----------|
    |                                                             |
    |                              |                              |
    | d.stack unwind/destroy       |                              |
    | e.clear dht_local_t in       |                              |
<-- |   dht_local_wipe             |                              |
    |                              |                              |

A tmp local->fd will be created for non-fd-based fops to ensure
safe access to fd or its member in subsequent procedures. Once this
linkfile was successfully created, its fd_count will be increased in
server cbk and passed to client if necessary. Then, between phase c
and phase e, linkfile will not be deleted by mistake since later
processes will know this linkfile is being used, instead of stale.
After phase e, this linkfile links to a datafile, so it is valid.

This change should also be applied during dht_lookup_everywhere_cbk.

Fixes: #4548
Signed-off-by: chenjinhao <[email protected]>

Author: chenjinhao

xlators/cluster/dht/src/dht-common.c

@@ -2426,12 +2426,20 @@ dht_lookup_everywhere_done(call_frame_t *frame, xlator_t *this)
                 gf_msg_debug(this->name, 0,
                              "No Cached was found and "
                              "unlink on hashed was skipped"
-                             " so performing now: %s",
-                             local->loc.path);
-                FRAME_SU_DO(frame, dht_local_t);
-                STACK_WIND(frame, dht_lookup_unlink_stale_linkto_cbk,
-                           hashed_subvol, hashed_subvol->fops->unlink,
-                           &local->loc, 0, local->xattr_req);
+                             " so performing now: %s. fd_count = %d",
+                             local->loc.path,
+                             local->skip_unlink.opend_fd_count);
+                if (local->skip_unlink.opend_fd_count == 0) {
+                    FRAME_SU_DO(frame, dht_local_t);
+                    STACK_WIND(frame, dht_lookup_unlink_stale_linkto_cbk,
+                               hashed_subvol, hashed_subvol->fops->unlink,
+                               &local->loc, 0, local->xattr_req);
+                } else {
+                    /* Skip linkfile deletion, this linkfile may be in used
+                     * because its fd_count > 0 */
+                    DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL,
+                                     NULL);
+                }
             }
 
         } else {

xlators/cluster/dht/src/dht-linkfile.c

@@ -8,6 +8,7 @@
   cases as published by the Free Software Foundation.
 */
 
+#include <fcntl.h>
 #include <glusterfs/compat.h>
 #include "dht-common.h"
 
@@ -45,7 +46,7 @@ dht_linkfile_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
 
 static int
 dht_linkfile_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, inode_t *inode,
+                        int op_ret, int op_errno, fd_t *fd, inode_t *inode,
                         struct iatt *stbuf, struct iatt *preparent,
                         struct iatt *postparent, dict_t *xdata)
 {
@@ -147,9 +148,15 @@ dht_linkfile_create(call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
     /* Always create as root:root. dht_linkfile_attr_heal fixes the
      * ownsership */
     FRAME_SU_DO(frame, dht_local_t);
+
+    if (!local->fd) {
+        local->fd = fd_create(loc->inode, frame->root->pid);
+        local->flags |= (O_CREAT | O_EXCL);
+    }
+
     STACK_WIND_COOKIE(frame, dht_linkfile_create_cbk, fromvol, fromvol,
-                      fromvol->fops->mknod, loc, S_IFREG | DHT_LINKFILE_MODE, 0,
-                      0, dict);
+                      fromvol->fops->create, loc, local->flags,
+                      S_IFREG | DHT_LINKFILE_MODE, 0, local->fd, dict);
 
     if (need_unref && dict)
         dict_unref(dict);