From cf07d2ea43e5c22149435ee9002cb737eac20eca Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Sun, 28 Feb 2010 23:20:19 -0500 Subject: nfsd4: simplify references to nfsd4 lease time Instead of accessing the lease time directly, some users call nfs4_lease_time(), and some a macro, NFSD_LEASE_TIME, defined as nfs4_lease_time(). Neither layer of indirection serves any purpose. Signed-off-by: J. Bruce Fields --- fs/nfsd/nfs4callback.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs/nfsd/nfs4callback.c') diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index 4bc22c763de7..ed12ad40828b 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -455,7 +455,7 @@ static struct rpc_program cb_program = { static int max_cb_time(void) { - return max(NFSD_LEASE_TIME/10, (time_t)1) * HZ; + return max(nfsd4_lease/10, (time_t)1) * HZ; } /* Reference counting, callback cleanup, etc., all look racy as heck. -- cgit From 227f98d98d2ed7929f41426adc21f57b927354a6 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 18 Feb 2010 08:27:24 -0800 Subject: nfsd4: preallocate nfs4_rpc_args Instead of allocating this small structure, just include it in the delegation. The nfsd4_callback structure isn't really necessary yet, but we plan to add to it all the information necessary to perform a callback. Signed-off-by: J. Bruce Fields --- fs/nfsd/nfs4callback.c | 18 +++--------------- fs/nfsd/state.h | 10 ++++++++++ 2 files changed, 13 insertions(+), 15 deletions(-) (limited to 'fs/nfsd/nfs4callback.c') diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index ed12ad40828b..b99c3f0f1d35 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -78,11 +78,6 @@ enum nfs_cb_opnum4 { cb_sequence_dec_sz + \ op_dec_sz) -struct nfs4_rpc_args { - void *args_op; - struct nfsd4_cb_sequence args_seq; -}; - /* * Generic encode routines from fs/nfs/nfs4xdr.c */ @@ -676,7 +671,7 @@ static void nfsd4_cb_recall_done(struct rpc_task *task, void *calldata) break; default: /* success, or error we can't handle */ - goto done; + return; } if (dp->dl_retries--) { rpc_delay(task, 2*HZ); @@ -687,8 +682,6 @@ static void nfsd4_cb_recall_done(struct rpc_task *task, void *calldata) atomic_set(&clp->cl_cb_conn.cb_set, 0); warn_no_callback_path(clp, task->tk_status); } -done: - kfree(task->tk_msg.rpc_argp); } static void nfsd4_cb_recall_release(void *calldata) @@ -714,24 +707,19 @@ nfsd4_cb_recall(struct nfs4_delegation *dp) { struct nfs4_client *clp = dp->dl_client; struct rpc_clnt *clnt = clp->cl_cb_conn.cb_client; - struct nfs4_rpc_args *args; + struct nfs4_rpc_args *args = &dp->dl_recall.cb_args; struct rpc_message msg = { .rpc_proc = &nfs4_cb_procedures[NFSPROC4_CLNT_CB_RECALL], .rpc_cred = callback_cred }; - int status = -ENOMEM; + int status; - args = kzalloc(sizeof(*args), GFP_KERNEL); - if (!args) - goto out; args->args_op = dp; msg.rpc_argp = args; dp->dl_retries = 1; status = rpc_call_async(clnt, &msg, RPC_TASK_SOFT, &nfsd4_cb_recall_ops, dp); -out: if (status) { - kfree(args); put_nfs4_client(clp); nfs4_put_delegation(dp); } diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index fefeae27f25e..b85437982a8d 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -70,6 +70,15 @@ struct nfsd4_cb_sequence { struct nfs4_client *cbs_clp; }; +struct nfs4_rpc_args { + void *args_op; + struct nfsd4_cb_sequence args_seq; +}; + +struct nfsd4_callback { + struct nfs4_rpc_args cb_args; +}; + struct nfs4_delegation { struct list_head dl_perfile; struct list_head dl_perclnt; @@ -86,6 +95,7 @@ struct nfs4_delegation { stateid_t dl_stateid; struct knfsd_fh dl_fh; int dl_retries; + struct nfsd4_callback dl_recall; }; /* client delegation callback info */ -- cgit From 9045b4b9f7f340f43de0cf687b5b52f6feaaa984 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Sun, 21 Feb 2010 17:53:04 -0800 Subject: nfsd4: remove probe task's reference on client Any null probe rpc will be synchronously destroyed by the rpc_shutdown_client() in expire_client(), so the rpc task cannot outlast the nfs4 client. Therefore there's no need for that task to hold a reference on the client. Signed-off-by: J. Bruce Fields --- fs/nfsd/nfs4callback.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'fs/nfsd/nfs4callback.c') diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index b99c3f0f1d35..91eb2ea9ef0a 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -509,7 +509,6 @@ static void nfsd4_cb_probe_done(struct rpc_task *task, void *calldata) warn_no_callback_path(clp, task->tk_status); else atomic_set(&clp->cl_cb_conn.cb_set, 1); - put_nfs4_client(clp); } static const struct rpc_call_ops nfsd4_cb_probe_ops = { @@ -542,10 +541,8 @@ void do_probe_callback(struct nfs4_client *clp) status = rpc_call_async(cb->cb_client, &msg, RPC_TASK_SOFT | RPC_TASK_SOFTCONN, &nfsd4_cb_probe_ops, (void *)clp); - if (status) { + if (status) warn_no_callback_path(clp, status); - put_nfs4_client(clp); - } } /* @@ -563,10 +560,6 @@ nfsd4_probe_callback(struct nfs4_client *clp) warn_no_callback_path(clp, status); return; } - - /* the task holds a reference to the nfs4_client struct */ - atomic_inc(&clp->cl_count); - do_probe_callback(clp); } -- cgit From b5a1a81e5c25fb6bb3fdc1812ba69ff6ab638fcf Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Wed, 3 Mar 2010 14:52:55 -0500 Subject: nfsd4: don't sleep in lease-break callback The NFSv4 server's fl_break callback can sleep (dropping the BKL), in order to allocate a new rpc task to send a recall to the client. As far as I can tell this doesn't cause any races in the current code, but the analysis is difficult. Also, the sleep here may complicate the move away from the BKL. So, just schedule some work to do the job for us instead. The work will later also prove useful for restarting a call after the callback information is changed. Signed-off-by: J. Bruce Fields --- fs/nfsd/nfs4callback.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++-- fs/nfsd/nfs4state.c | 34 +++++++++++++++---------------- fs/nfsd/state.h | 5 +++++ 3 files changed, 73 insertions(+), 20 deletions(-) (limited to 'fs/nfsd/nfs4callback.c') diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index 91eb2ea9ef0a..e078c747f49d 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -32,6 +32,7 @@ */ #include +#include #include "nfsd.h" #include "state.h" @@ -692,11 +693,41 @@ static const struct rpc_call_ops nfsd4_cb_recall_ops = { .rpc_release = nfsd4_cb_recall_release, }; +static struct workqueue_struct *callback_wq; + +int nfsd4_create_callback_queue(void) +{ + callback_wq = create_singlethread_workqueue("nfsd4_callbacks"); + if (!callback_wq) + return -ENOMEM; + return 0; +} + +void nfsd4_destroy_callback_queue(void) +{ + destroy_workqueue(callback_wq); +} + +void nfsd4_set_callback_client(struct nfs4_client *clp, struct rpc_clnt +*new) +{ + struct rpc_clnt *old = clp->cl_cb_conn.cb_client; + + clp->cl_cb_conn.cb_client = new; + /* + * After this, any work that saw the old value of cb_client will + * be gone: + */ + flush_workqueue(callback_wq); + /* So we can safely shut it down: */ + if (old) + rpc_shutdown_client(old); +} + /* * called with dp->dl_count inc'ed. */ -void -nfsd4_cb_recall(struct nfs4_delegation *dp) +static void _nfsd4_cb_recall(struct nfs4_delegation *dp) { struct nfs4_client *clp = dp->dl_client; struct rpc_clnt *clnt = clp->cl_cb_conn.cb_client; @@ -707,6 +738,9 @@ nfsd4_cb_recall(struct nfs4_delegation *dp) }; int status; + if (clnt == NULL) + return; /* Client is shutting down; give up. */ + args->args_op = dp; msg.rpc_argp = args; dp->dl_retries = 1; @@ -717,3 +751,19 @@ nfsd4_cb_recall(struct nfs4_delegation *dp) nfs4_put_delegation(dp); } } + +void nfsd4_do_callback_rpc(struct work_struct *w) +{ + /* XXX: for now, just send off delegation recall. */ + /* In future, generalize to handle any sort of callback. */ + struct nfsd4_callback *c = container_of(w, struct nfsd4_callback, cb_work); + struct nfs4_delegation *dp = container_of(c, struct nfs4_delegation, dl_recall); + + _nfsd4_cb_recall(dp); +} + + +void nfsd4_cb_recall(struct nfs4_delegation *dp) +{ + queue_work(callback_wq, &dp->dl_recall.cb_work); +} diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 5051ade30dfb..adc51d10d435 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -198,6 +198,7 @@ alloc_init_deleg(struct nfs4_client *clp, struct nfs4_stateid *stp, struct svc_f atomic_set(&dp->dl_count, 1); list_add(&dp->dl_perfile, &fp->fi_delegations); list_add(&dp->dl_perclnt, &clp->cl_delegations); + INIT_WORK(&dp->dl_recall.cb_work, nfsd4_do_callback_rpc); return dp; } @@ -679,21 +680,6 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name) return clp; } -static void -shutdown_callback_client(struct nfs4_client *clp) -{ - struct rpc_clnt *clnt = clp->cl_cb_conn.cb_client; - - if (clnt) { - /* - * Callback threads take a reference on the client, so there - * should be no outstanding callbacks at this point. - */ - clp->cl_cb_conn.cb_client = NULL; - rpc_shutdown_client(clnt); - } -} - static inline void free_client(struct nfs4_client *clp) { @@ -746,7 +732,7 @@ expire_client(struct nfs4_client *clp) se_perclnt); release_session(ses); } - shutdown_callback_client(clp); + nfsd4_set_callback_client(clp, NULL); if (clp->cl_cb_xprt) svc_xprt_put(clp->cl_cb_xprt); put_nfs4_client(clp); @@ -1392,7 +1378,7 @@ nfsd4_destroy_session(struct svc_rqst *r, spin_unlock(&sessionid_lock); /* wait for callbacks */ - shutdown_callback_client(ses->se_client); + nfsd4_set_callback_client(ses->se_client, NULL); nfsd4_put_session(ses); status = nfs_ok; out: @@ -4004,16 +3990,27 @@ set_max_delegations(void) static int __nfs4_state_start(void) { + int ret; + boot_time = get_seconds(); locks_start_grace(&nfsd4_manager); printk(KERN_INFO "NFSD: starting %ld-second grace period\n", nfsd4_grace); + ret = set_callback_cred(); + if (ret) + return -ENOMEM; laundry_wq = create_singlethread_workqueue("nfsd4"); if (laundry_wq == NULL) return -ENOMEM; + ret = nfsd4_create_callback_queue(); + if (ret) + goto out_free_laundry; queue_delayed_work(laundry_wq, &laundromat_work, nfsd4_grace * HZ); set_max_delegations(); - return set_callback_cred(); + return 0; +out_free_laundry: + destroy_workqueue(laundry_wq); + return ret; } int @@ -4075,6 +4072,7 @@ nfs4_state_shutdown(void) nfs4_lock_state(); nfs4_release_reclaim(); __nfs4_state_shutdown(); + nfsd4_destroy_callback_queue(); nfs4_unlock_state(); } diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index b85437982a8d..c4c92aea8f39 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -77,6 +77,7 @@ struct nfs4_rpc_args { struct nfsd4_callback { struct nfs4_rpc_args cb_args; + struct work_struct cb_work; }; struct nfs4_delegation { @@ -391,7 +392,11 @@ extern void put_nfs4_client(struct nfs4_client *clp); extern void nfs4_free_stateowner(struct kref *kref); extern int set_callback_cred(void); extern void nfsd4_probe_callback(struct nfs4_client *clp); +extern void nfsd4_do_callback_rpc(struct work_struct *); extern void nfsd4_cb_recall(struct nfs4_delegation *dp); +extern int nfsd4_create_callback_queue(void); +extern void nfsd4_destroy_callback_queue(void); +extern void nfsd4_set_callback_client(struct nfs4_client *, struct rpc_clnt *); extern void nfs4_put_delegation(struct nfs4_delegation *dp); extern __be32 nfs4_make_rec_clidname(char *clidname, struct xdr_netobj *clname); extern void nfsd4_init_recdir(char *recdir_name); -- cgit From b12a05cbdfdf7e4d8cbe8fa78e995f971420086b Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 4 Mar 2010 11:32:59 -0500 Subject: nfsd4: cl_count is unused Now that the shutdown sequence guarantees callbacks are shut down before the client is destroyed, we no longer have a use for cl_count. We'll probably reinstate a reference count on the client some day, but it will be held by users other than callbacks. Signed-off-by: J. Bruce Fields --- fs/nfsd/nfs4callback.c | 6 +----- fs/nfsd/nfs4state.c | 11 +---------- fs/nfsd/state.h | 2 -- 3 files changed, 2 insertions(+), 17 deletions(-) (limited to 'fs/nfsd/nfs4callback.c') diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index e078c747f49d..5856fc8adb70 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -681,10 +681,8 @@ static void nfsd4_cb_recall_done(struct rpc_task *task, void *calldata) static void nfsd4_cb_recall_release(void *calldata) { struct nfs4_delegation *dp = calldata; - struct nfs4_client *clp = dp->dl_client; nfs4_put_delegation(dp); - put_nfs4_client(clp); } static const struct rpc_call_ops nfsd4_cb_recall_ops = { @@ -746,10 +744,8 @@ static void _nfsd4_cb_recall(struct nfs4_delegation *dp) dp->dl_retries = 1; status = rpc_call_async(clnt, &msg, RPC_TASK_SOFT, &nfsd4_cb_recall_ops, dp); - if (status) { - put_nfs4_client(clp); + if (status) nfs4_put_delegation(dp); - } } void nfsd4_do_callback_rpc(struct work_struct *w) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index adc51d10d435..cf650cbb814b 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -690,13 +690,6 @@ free_client(struct nfs4_client *clp) kfree(clp); } -void -put_nfs4_client(struct nfs4_client *clp) -{ - if (atomic_dec_and_test(&clp->cl_count)) - free_client(clp); -} - static void expire_client(struct nfs4_client *clp) { @@ -735,7 +728,7 @@ expire_client(struct nfs4_client *clp) nfsd4_set_callback_client(clp, NULL); if (clp->cl_cb_xprt) svc_xprt_put(clp->cl_cb_xprt); - put_nfs4_client(clp); + free_client(clp); } static void copy_verf(struct nfs4_client *target, nfs4_verifier *source) @@ -821,7 +814,6 @@ static struct nfs4_client *create_client(struct xdr_netobj name, char *recdir, } memcpy(clp->cl_recdir, recdir, HEXDIR_LEN); - atomic_set(&clp->cl_count, 1); atomic_set(&clp->cl_cb_conn.cb_set, 0); INIT_LIST_HEAD(&clp->cl_idhash); INIT_LIST_HEAD(&clp->cl_strhash); @@ -2010,7 +2002,6 @@ void nfsd_break_deleg_cb(struct file_lock *fl) * lock) we know the server hasn't removed the lease yet, we know * it's safe to take a reference: */ atomic_inc(&dp->dl_count); - atomic_inc(&dp->dl_client->cl_count); spin_lock(&recall_lock); list_add_tail(&dp->dl_recall_lru, &del_recall_lru); diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index c4c92aea8f39..cef20abf330c 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -224,7 +224,6 @@ struct nfs4_client { clientid_t cl_clientid; /* generated by server */ nfs4_verifier cl_confirm; /* generated by server */ struct nfs4_cb_conn cl_cb_conn; /* callback info */ - atomic_t cl_count; /* ref count */ u32 cl_firststate; /* recovery dir creation */ /* for nfs41 */ @@ -388,7 +387,6 @@ extern void nfs4_lock_state(void); extern void nfs4_unlock_state(void); extern int nfs4_in_grace(void); extern __be32 nfs4_check_open_reclaim(clientid_t *clid); -extern void put_nfs4_client(struct nfs4_client *clp); extern void nfs4_free_stateowner(struct kref *kref); extern int set_callback_cred(void); extern void nfsd4_probe_callback(struct nfs4_client *clp); -- cgit From 2bf23875f55af6038a5d1c164a52cec4c24609ba Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Mon, 8 Mar 2010 12:37:27 -0500 Subject: nfsd4: rearrange cb data structures Mainly I just want to separate the arguments used for setting up the tcp client from the rest. Signed-off-by: J. Bruce Fields --- fs/nfsd/nfs4callback.c | 25 ++++++++++++------------- fs/nfsd/nfs4state.c | 20 ++++++++++---------- fs/nfsd/state.h | 11 ++++++----- 3 files changed, 28 insertions(+), 28 deletions(-) (limited to 'fs/nfsd/nfs4callback.c') diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index 5856fc8adb70..d6c46a9de422 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -455,7 +455,7 @@ static int max_cb_time(void) } /* Reference counting, callback cleanup, etc., all look racy as heck. - * And why is cb_set an atomic? */ + * And why is cl_cb_set an atomic? */ int setup_callback_client(struct nfs4_client *clp) { @@ -481,7 +481,7 @@ int setup_callback_client(struct nfs4_client *clp) if (!clp->cl_principal && (clp->cl_flavor >= RPC_AUTH_GSS_KRB5)) return -EINVAL; if (cb->cb_minorversion) { - args.bc_xprt = clp->cl_cb_xprt; + args.bc_xprt = clp->cl_cb_conn.cb_xprt; args.protocol = XPRT_TRANSPORT_BC_TCP; } /* Create RPC client */ @@ -491,7 +491,7 @@ int setup_callback_client(struct nfs4_client *clp) PTR_ERR(client)); return PTR_ERR(client); } - cb->cb_client = client; + clp->cl_cb_client = client; return 0; } @@ -509,7 +509,7 @@ static void nfsd4_cb_probe_done(struct rpc_task *task, void *calldata) if (task->tk_status) warn_no_callback_path(clp, task->tk_status); else - atomic_set(&clp->cl_cb_conn.cb_set, 1); + atomic_set(&clp->cl_cb_set, 1); } static const struct rpc_call_ops nfsd4_cb_probe_ops = { @@ -531,7 +531,6 @@ int set_callback_cred(void) void do_probe_callback(struct nfs4_client *clp) { - struct nfs4_cb_conn *cb = &clp->cl_cb_conn; struct rpc_message msg = { .rpc_proc = &nfs4_cb_procedures[NFSPROC4_CLNT_CB_NULL], .rpc_argp = clp, @@ -539,7 +538,7 @@ void do_probe_callback(struct nfs4_client *clp) }; int status; - status = rpc_call_async(cb->cb_client, &msg, + status = rpc_call_async(clp->cl_cb_client, &msg, RPC_TASK_SOFT | RPC_TASK_SOFTCONN, &nfsd4_cb_probe_ops, (void *)clp); if (status) @@ -554,7 +553,7 @@ nfsd4_probe_callback(struct nfs4_client *clp) { int status; - BUG_ON(atomic_read(&clp->cl_cb_conn.cb_set)); + BUG_ON(atomic_read(&clp->cl_cb_set)); status = setup_callback_client(clp); if (status) { @@ -656,7 +655,7 @@ static void nfsd4_cb_recall_done(struct rpc_task *task, void *calldata) switch (task->tk_status) { case -EIO: /* Network partition? */ - atomic_set(&clp->cl_cb_conn.cb_set, 0); + atomic_set(&clp->cl_cb_set, 0); warn_no_callback_path(clp, task->tk_status); case -EBADHANDLE: case -NFS4ERR_BAD_STATEID: @@ -673,7 +672,7 @@ static void nfsd4_cb_recall_done(struct rpc_task *task, void *calldata) rpc_restart_call(task); return; } else { - atomic_set(&clp->cl_cb_conn.cb_set, 0); + atomic_set(&clp->cl_cb_set, 0); warn_no_callback_path(clp, task->tk_status); } } @@ -709,11 +708,11 @@ void nfsd4_destroy_callback_queue(void) void nfsd4_set_callback_client(struct nfs4_client *clp, struct rpc_clnt *new) { - struct rpc_clnt *old = clp->cl_cb_conn.cb_client; + struct rpc_clnt *old = clp->cl_cb_client; - clp->cl_cb_conn.cb_client = new; + clp->cl_cb_client = new; /* - * After this, any work that saw the old value of cb_client will + * After this, any work that saw the old value of cl_cb_client will * be gone: */ flush_workqueue(callback_wq); @@ -728,7 +727,7 @@ void nfsd4_set_callback_client(struct nfs4_client *clp, struct rpc_clnt static void _nfsd4_cb_recall(struct nfs4_delegation *dp) { struct nfs4_client *clp = dp->dl_client; - struct rpc_clnt *clnt = clp->cl_cb_conn.cb_client; + struct rpc_clnt *clnt = clp->cl_cb_client; struct nfs4_rpc_args *args = &dp->dl_recall.cb_args; struct rpc_message msg = { .rpc_proc = &nfs4_cb_procedures[NFSPROC4_CLNT_CB_RECALL], diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index cf650cbb814b..59c9bd4c89e1 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -726,8 +726,8 @@ expire_client(struct nfs4_client *clp) release_session(ses); } nfsd4_set_callback_client(clp, NULL); - if (clp->cl_cb_xprt) - svc_xprt_put(clp->cl_cb_xprt); + if (clp->cl_cb_conn.cb_xprt) + svc_xprt_put(clp->cl_cb_conn.cb_xprt); free_client(clp); } @@ -814,7 +814,7 @@ static struct nfs4_client *create_client(struct xdr_netobj name, char *recdir, } memcpy(clp->cl_recdir, recdir, HEXDIR_LEN); - atomic_set(&clp->cl_cb_conn.cb_set, 0); + atomic_set(&clp->cl_cb_set, 0); INIT_LIST_HEAD(&clp->cl_idhash); INIT_LIST_HEAD(&clp->cl_strhash); INIT_LIST_HEAD(&clp->cl_openowners); @@ -1302,8 +1302,8 @@ nfsd4_create_session(struct svc_rqst *rqstp, move_to_confirmed(unconf); if (cr_ses->flags & SESSION4_BACK_CHAN) { - unconf->cl_cb_xprt = rqstp->rq_xprt; - svc_xprt_get(unconf->cl_cb_xprt); + unconf->cl_cb_conn.cb_xprt = rqstp->rq_xprt; + svc_xprt_get(rqstp->rq_xprt); rpc_copy_addr( (struct sockaddr *)&unconf->cl_cb_conn.cb_addr, sa); @@ -1607,7 +1607,7 @@ nfsd4_setclientid_confirm(struct svc_rqst *rqstp, else { /* XXX: We just turn off callbacks until we can handle * change request correctly. */ - atomic_set(&conf->cl_cb_conn.cb_set, 0); + atomic_set(&conf->cl_cb_set, 0); expire_client(unconf); status = nfs_ok; @@ -2320,7 +2320,7 @@ nfs4_open_delegation(struct svc_fh *fh, struct nfsd4_open *open, struct nfs4_sta { struct nfs4_delegation *dp; struct nfs4_stateowner *sop = stp->st_stateowner; - struct nfs4_cb_conn *cb = &sop->so_client->cl_cb_conn; + int cb_up = atomic_read(&sop->so_client->cl_cb_set); struct file_lock fl, *flp = &fl; int status, flag = 0; @@ -2328,7 +2328,7 @@ nfs4_open_delegation(struct svc_fh *fh, struct nfsd4_open *open, struct nfs4_sta open->op_recall = 0; switch (open->op_claim_type) { case NFS4_OPEN_CLAIM_PREVIOUS: - if (!atomic_read(&cb->cb_set)) + if (!cb_up) open->op_recall = 1; flag = open->op_delegate_type; if (flag == NFS4_OPEN_DELEGATE_NONE) @@ -2339,7 +2339,7 @@ nfs4_open_delegation(struct svc_fh *fh, struct nfsd4_open *open, struct nfs4_sta * had the chance to reclaim theirs.... */ if (locks_in_grace()) goto out; - if (!atomic_read(&cb->cb_set) || !sop->so_confirmed) + if (!cb_up || !sop->so_confirmed) goto out; if (open->op_share_access & NFS4_SHARE_ACCESS_WRITE) flag = NFS4_OPEN_DELEGATE_WRITE; @@ -2510,7 +2510,7 @@ nfsd4_renew(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, renew_client(clp); status = nfserr_cb_path_down; if (!list_empty(&clp->cl_delegations) - && !atomic_read(&clp->cl_cb_conn.cb_set)) + && !atomic_read(&clp->cl_cb_set)) goto out; status = nfs_ok; out: diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index cef20abf330c..cf43812e6da5 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -107,9 +107,7 @@ struct nfs4_cb_conn { u32 cb_prog; u32 cb_minorversion; u32 cb_ident; /* minorversion 0 only */ - /* RPC client info */ - atomic_t cb_set; /* successful CB_NULL call */ - struct rpc_clnt * cb_client; + struct svc_xprt *cb_xprt; /* minorversion 1 only */ }; /* Maximum number of slots per session. 160 is useful for long haul TCP */ @@ -223,9 +221,13 @@ struct nfs4_client { struct svc_cred cl_cred; /* setclientid principal */ clientid_t cl_clientid; /* generated by server */ nfs4_verifier cl_confirm; /* generated by server */ - struct nfs4_cb_conn cl_cb_conn; /* callback info */ u32 cl_firststate; /* recovery dir creation */ + /* for v4.0 and v4.1 callbacks: */ + struct nfs4_cb_conn cl_cb_conn; + struct rpc_clnt *cl_cb_client; + atomic_t cl_cb_set; + /* for nfs41 */ struct list_head cl_sessions; struct nfsd4_clid_slot cl_cs_slot; /* create_session slot */ @@ -236,7 +238,6 @@ struct nfs4_client { /* We currently support a single back channel with a single slot */ unsigned long cl_cb_slot_busy; u32 cl_cb_seq_nr; - struct svc_xprt *cl_cb_xprt; /* 4.1 callback transport */ struct rpc_wait_queue cl_cb_waitq; /* backchannel callers may */ /* wait here for slots */ }; -- cgit From 4b21d0defcc9680da8a694e92d5fe8eb668c2c0b Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Sun, 7 Mar 2010 23:39:01 -0500 Subject: nfsd4: allow 4.0 clients to change callback path The rfc allows a client to change the callback parameters, but we didn't previously implement it. Teach the callbacks to rerun themselves (by placing themselves on a workqueue) when they recognize that their rpc task has been killed and that the callback connection has changed. Then we can change the callback connection by setting up a new rpc client, modifying the nfs4 client to point at it, waiting for any work in progress to complete, and then shutting down the old client. Signed-off-by: J. Bruce Fields --- fs/nfsd/nfs4callback.c | 29 ++++++++++++++++++++--------- fs/nfsd/nfs4state.c | 7 +++---- fs/nfsd/state.h | 2 +- 3 files changed, 24 insertions(+), 14 deletions(-) (limited to 'fs/nfsd/nfs4callback.c') diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index d6c46a9de422..ea77aa63754a 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -457,9 +457,8 @@ static int max_cb_time(void) /* Reference counting, callback cleanup, etc., all look racy as heck. * And why is cl_cb_set an atomic? */ -int setup_callback_client(struct nfs4_client *clp) +int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *cb) { - struct nfs4_cb_conn *cb = &clp->cl_cb_conn; struct rpc_timeout timeparms = { .to_initval = max_cb_time(), .to_retries = 0, @@ -481,7 +480,7 @@ int setup_callback_client(struct nfs4_client *clp) if (!clp->cl_principal && (clp->cl_flavor >= RPC_AUTH_GSS_KRB5)) return -EINVAL; if (cb->cb_minorversion) { - args.bc_xprt = clp->cl_cb_conn.cb_xprt; + args.bc_xprt = cb->cb_xprt; args.protocol = XPRT_TRANSPORT_BC_TCP; } /* Create RPC client */ @@ -491,7 +490,7 @@ int setup_callback_client(struct nfs4_client *clp) PTR_ERR(client)); return PTR_ERR(client); } - clp->cl_cb_client = client; + nfsd4_set_callback_client(clp, client); return 0; } @@ -548,14 +547,13 @@ void do_probe_callback(struct nfs4_client *clp) /* * Set up the callback client and put a NFSPROC4_CB_NULL on the wire... */ -void -nfsd4_probe_callback(struct nfs4_client *clp) +void nfsd4_probe_callback(struct nfs4_client *clp, struct nfs4_cb_conn *cb) { int status; BUG_ON(atomic_read(&clp->cl_cb_set)); - status = setup_callback_client(clp); + status = setup_callback_client(clp, cb); if (status) { warn_no_callback_path(clp, status); return; @@ -645,18 +643,32 @@ static void nfsd4_cb_done(struct rpc_task *task, void *calldata) } } + static void nfsd4_cb_recall_done(struct rpc_task *task, void *calldata) { struct nfs4_delegation *dp = calldata; struct nfs4_client *clp = dp->dl_client; + struct rpc_clnt *current_rpc_client = clp->cl_cb_client; nfsd4_cb_done(task, calldata); + if (current_rpc_client == NULL) { + /* We're shutting down; give up. */ + /* XXX: err, or is it ok just to fall through + * and rpc_restart_call? */ + return; + } + switch (task->tk_status) { case -EIO: /* Network partition? */ atomic_set(&clp->cl_cb_set, 0); warn_no_callback_path(clp, task->tk_status); + if (current_rpc_client != task->tk_client) { + /* queue a callback on the new connection: */ + nfsd4_cb_recall(dp); + return; + } case -EBADHANDLE: case -NFS4ERR_BAD_STATEID: /* Race: client probably got cb_recall @@ -705,8 +717,7 @@ void nfsd4_destroy_callback_queue(void) destroy_workqueue(callback_wq); } -void nfsd4_set_callback_client(struct nfs4_client *clp, struct rpc_clnt -*new) +void nfsd4_set_callback_client(struct nfs4_client *clp, struct rpc_clnt *new) { struct rpc_clnt *old = clp->cl_cb_client; diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 59c9bd4c89e1..4300d9ffe95f 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -1312,7 +1312,7 @@ nfsd4_create_session(struct svc_rqst *rqstp, cstate->minorversion; unconf->cl_cb_conn.cb_prog = cr_ses->callback_prog; unconf->cl_cb_seq_nr = 1; - nfsd4_probe_callback(unconf); + nfsd4_probe_callback(unconf, &unconf->cl_cb_conn); } conf = unconf; } else { @@ -1605,9 +1605,8 @@ nfsd4_setclientid_confirm(struct svc_rqst *rqstp, if (!same_creds(&conf->cl_cred, &unconf->cl_cred)) status = nfserr_clid_inuse; else { - /* XXX: We just turn off callbacks until we can handle - * change request correctly. */ atomic_set(&conf->cl_cb_set, 0); + nfsd4_probe_callback(conf, &unconf->cl_cb_conn); expire_client(unconf); status = nfs_ok; @@ -1641,7 +1640,7 @@ nfsd4_setclientid_confirm(struct svc_rqst *rqstp, } move_to_confirmed(unconf); conf = unconf; - nfsd4_probe_callback(conf); + nfsd4_probe_callback(conf, &conf->cl_cb_conn); status = nfs_ok; } } else if ((!conf || (conf && !same_verf(&conf->cl_confirm, &confirm))) diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index cf43812e6da5..98836fd87f69 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -390,7 +390,7 @@ extern int nfs4_in_grace(void); extern __be32 nfs4_check_open_reclaim(clientid_t *clid); extern void nfs4_free_stateowner(struct kref *kref); extern int set_callback_cred(void); -extern void nfsd4_probe_callback(struct nfs4_client *clp); +extern void nfsd4_probe_callback(struct nfs4_client *clp, struct nfs4_cb_conn *); extern void nfsd4_do_callback_rpc(struct work_struct *); extern void nfsd4_cb_recall(struct nfs4_delegation *dp); extern int nfsd4_create_callback_queue(void); -- cgit From ab707e156593ff7fffd615757332dbff6616836a Mon Sep 17 00:00:00 2001 From: Benny Halevy Date: Wed, 12 May 2010 00:14:06 +0300 Subject: nfsd4: nfsd4_destroy_session must set callback client under the state lock nfsd4_set_callback_client must be called under the state lock to atomically set or unset the callback client and shutting down the previous one. Signed-off-by: Benny Halevy Signed-off-by: J. Bruce Fields --- fs/nfsd/nfs4callback.c | 1 + fs/nfsd/nfs4state.c | 2 ++ 2 files changed, 3 insertions(+) (limited to 'fs/nfsd/nfs4callback.c') diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index 1d5051d46b46..77bc9d3c80fd 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -718,6 +718,7 @@ void nfsd4_destroy_callback_queue(void) destroy_workqueue(callback_wq); } +/* must be called under the state lock */ void nfsd4_set_callback_client(struct nfs4_client *clp, struct rpc_clnt *new) { struct rpc_clnt *old = clp->cl_cb_client; diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index cc0e9117dd16..ede9dde52fe8 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -1427,8 +1427,10 @@ nfsd4_destroy_session(struct svc_rqst *r, unhash_session(ses); spin_unlock(&client_lock); + nfs4_lock_state(); /* wait for callbacks */ nfsd4_set_callback_client(ses->se_client, NULL); + nfs4_unlock_state(); nfsd4_put_session(ses); status = nfs_ok; out: -- cgit From b7299f44394336f51b526247a870d47d28f4f97c Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Fri, 14 May 2010 17:57:35 -0400 Subject: nfs4: minor callback code simplification, comment Note the position in the version array doesn't have to match the actual rpc version number--to me it seems clearer to maintain the distinction. Also document choice of rpc callback version number, as discussed in e.g. http://www.ietf.org/mail-archive/web/nfsv4/current/msg07985.html and followups. Signed-off-by: J. Bruce Fields --- fs/nfsd/nfs4callback.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'fs/nfsd/nfs4callback.c') diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index 77bc9d3c80fd..eb78e7e22077 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -424,13 +424,19 @@ static struct rpc_procinfo nfs4_cb_procedures[] = { }; static struct rpc_version nfs_cb_version4 = { +/* + * Note on the callback rpc program version number: despite language in rfc + * 5661 section 18.36.3 requiring servers to use 4 in this field, the + * official xdr descriptions for both 4.0 and 4.1 specify version 1, and + * in practice that appears to be what implementations use. The section + * 18.36.3 language is expected to be fixed in an erratum. + */ .number = 1, .nrprocs = ARRAY_SIZE(nfs4_cb_procedures), .procs = nfs4_cb_procedures }; static struct rpc_version * nfs_cb_version[] = { - NULL, &nfs_cb_version4, }; @@ -471,7 +477,7 @@ int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *cb) .timeout = &timeparms, .program = &cb_program, .prognumber = cb->cb_prog, - .version = nfs_cb_version[1]->number, + .version = 0, .authflavor = clp->cl_flavor, .flags = (RPC_CLNT_CREATE_NOPING | RPC_CLNT_CREATE_QUIET), .client_name = clp->cl_principal, -- cgit