diff --git a/src/Protocols/NFS/nfs4_op_reclaim_complete.c b/src/Protocols/NFS/nfs4_op_reclaim_complete.c index 46de77efde..69a89b1dbf 100644 --- a/src/Protocols/NFS/nfs4_op_reclaim_complete.c +++ b/src/Protocols/NFS/nfs4_op_reclaim_complete.c @@ -49,6 +49,7 @@ #include "nfs_proto_functions.h" #include "nfs_file_handle.h" #include "sal_data.h" +#include "sal_functions.h" /** * @@ -101,6 +102,7 @@ int nfs4_op_reclaim_complete(struct nfs_argop4 *op, compound_data_t *data, if (!arg_RECLAIM_COMPLETE4->rca_one_fs) { data->session->clientid_record->cid_cb.v41. cid_reclaim_complete = true; + nfs4_recovery_reclaim_complete(data->session->clientid_record); } return res_RECLAIM_COMPLETE4->rcr_status; diff --git a/src/SAL/nfs4_recovery.c b/src/SAL/nfs4_recovery.c index fd69a4f186..166ca2b534 100644 --- a/src/SAL/nfs4_recovery.c +++ b/src/SAL/nfs4_recovery.c @@ -46,6 +46,8 @@ time_t current_grace; pthread_mutex_t grace_mutex = PTHREAD_MUTEX_INITIALIZER; /*< Mutex */ struct glist_head clid_list = GLIST_HEAD_INIT(clid_list); /*< Clients */ struct nfs4_recovery_backend *recovery_backend; +static int clid_count; /* protected by grace_mutex */ +static int32_t reclaim_completes; /* atomic */ static void nfs4_recovery_load_clids_nolock(nfs_grace_start_t *gsp); static void nfs_release_nlm_state(char *release_ip); @@ -56,7 +58,9 @@ clid_entry_t *nfs4_add_clid_entry(char *cl_name) clid_entry_t *new_ent = gsh_malloc(sizeof(clid_entry_t)); strcpy(new_ent->cl_name, cl_name); + new_ent->cl_reclaim_complete = false; glist_add(&clid_list, &new_ent->cl_list); + ++clid_count; return new_ent; } @@ -78,6 +82,7 @@ void nfs4_cleanup_clid_entrys(void) cl_list)) != NULL) { glist_del(&clid_entry->cl_list); gsh_free(clid_entry); + --clid_count; } } @@ -149,19 +154,34 @@ int nfs_in_grace(void) { int in_grace; static int last_grace = -1; + int32_t rc_count = 0; if (nfs_param.nfsv4_param.graceless) return 0; - in_grace = ((atomic_fetch_time_t(¤t_grace) + - nfs_param.nfsv4_param.grace_period) > time(NULL)); + + in_grace = 1; + + /* + * If we know there are no NLM clients, then we can consider the grace + * period done when all previous clients have sent a RECLAIM_COMPLETE. + */ + rc_count = atomic_fetch_int32_t(&reclaim_completes); + if (!nfs_param.core_param.enable_NLM) + in_grace = (rc_count != clid_count); + + /* Otherwise, wait for the timeout */ + if (in_grace) + in_grace = ((atomic_fetch_time_t(¤t_grace) + + nfs_param.nfsv4_param.grace_period) > time(NULL)); if (in_grace != last_grace) { - LogEvent(COMPONENT_STATE, "NFS Server Now %s", + LogEvent(COMPONENT_STATE, "NFS Server Now %s ", in_grace ? "IN GRACE" : "NOT IN GRACE"); last_grace = in_grace; } else if (in_grace) { - LogDebug(COMPONENT_STATE, "NFS Server IN GRACE"); + LogDebug(COMPONENT_STATE, "NFS Server IN GRACE (%d:%d)", + clid_count, rc_count); } return in_grace; @@ -213,6 +233,48 @@ static bool check_clid(nfs_client_id_t *clientid, clid_entry_t *clid_ent) return ret; } +void nfs4_recovery_reclaim_complete(nfs_client_id_t *clientid) +{ + struct glist_head *node; + + /* If there are no clients */ + if (clid_count == 0) + return; + + /* + * Loop through the list and try to find a matching client that hasn't + * yet sent a reclaim_complete. If we find it, mark its clid_ent + * record, and increment the reclaim_completes counter. + */ + PTHREAD_MUTEX_lock(&grace_mutex); + glist_for_each(node, &clid_list) { + clid_entry_t *clid_ent = glist_entry(node, clid_entry_t, + cl_list); + + /* Skip any that have already sent a reclaim_complete */ + if (clid_ent->cl_reclaim_complete) + continue; + + if (check_clid(clientid, clid_ent)) { + if (isDebug(COMPONENT_CLIENTID)) { + char str[LOG_BUFF_LEN] = "\0"; + struct display_buffer dspbuf = { + sizeof(str), str, str}; + + display_client_id_rec(&dspbuf, clientid); + + LogFullDebug(COMPONENT_CLIENTID, + "RECLAIM_COMPLETE for %s", + str); + } + clid_ent->cl_reclaim_complete = true; + atomic_inc_int32_t(&reclaim_completes); + break; + } + } + PTHREAD_MUTEX_unlock(&grace_mutex); +} + /** * @brief Determine whether or not this client may reclaim state * @@ -230,7 +292,7 @@ void nfs4_chk_clid_impl(nfs_client_id_t *clientid, clid_entry_t **clid_ent_arg) clientid->cid_clientid); /* If there were no clients at time of restart, we're done */ - if (glist_empty(&clid_list)) + if (clid_count == 0) return; /* diff --git a/src/include/sal_data.h b/src/include/sal_data.h index c5eb4c2d8a..c5b3d004e9 100644 --- a/src/include/sal_data.h +++ b/src/include/sal_data.h @@ -216,6 +216,7 @@ typedef struct rdel_fh { typedef struct clid_entry { struct glist_head cl_list; /*< Link in the list */ struct glist_head cl_rfh_list; + bool cl_reclaim_complete; char cl_name[PATH_MAX]; /*< Client name */ } clid_entry_t; diff --git a/src/include/sal_functions.h b/src/include/sal_functions.h index 88df52eeba..8b5b1ad92d 100644 --- a/src/include/sal_functions.h +++ b/src/include/sal_functions.h @@ -968,6 +968,7 @@ void nfs4_start_grace(nfs_grace_start_t *gsp); int nfs_in_grace(void); void nfs4_add_clid(nfs_client_id_t *); void nfs4_rm_clid(nfs_client_id_t *); +void nfs4_recovery_reclaim_complete(nfs_client_id_t *clientid); void nfs4_chk_clid(nfs_client_id_t *); void nfs4_recovery_load_clids(nfs_grace_start_t *gsp); void nfs4_recovery_cleanup(void);