From a45e27e28023e9a40f796daace5b1c0a2c02651b Mon Sep 17 00:00:00 2001 From: shaver Date: Thu, 4 Dec 2003 20:07:30 +0000 Subject: [PATCH] b=2329: move osc_rpcd into ptlrpc as ptlrpcd, for non-OSC applications. Largely mechanical, plus a tiny Makefile.am cleanup in ptlrpc. r=zab --- lustre/include/linux/lustre_net.h | 5 + lustre/osc/Makefile.am | 4 +- lustre/osc/osc_create.c | 2 +- lustre/osc/osc_internal.h | 3 - lustre/osc/osc_request.c | 10 +- lustre/osc/osc_rpcd.c | 247 -------------------------------------- lustre/ptlrpc/Makefile.am | 14 +-- lustre/ptlrpc/ptlrpc_module.c | 5 + lustre/ptlrpc/ptlrpcd.c | 240 ++++++++++++++++++++++++++++++++++++ 9 files changed, 265 insertions(+), 265 deletions(-) delete mode 100644 lustre/osc/osc_rpcd.c create mode 100644 lustre/ptlrpc/ptlrpcd.c diff --git a/lustre/include/linux/lustre_net.h b/lustre/include/linux/lustre_net.h index d95ae9c..bf6aefc 100644 --- a/lustre/include/linux/lustre_net.h +++ b/lustre/include/linux/lustre_net.h @@ -551,6 +551,11 @@ int client_disconnect_export(struct obd_export *exp, int failover); int ptlrpc_pinger_add_import(struct obd_import *imp); int ptlrpc_pinger_del_import(struct obd_import *imp); +/* ptlrpc/ptlrpcd.c */ +void ptlrpcd_add_req(struct ptlrpc_request *req); +int ptlrpcd_addref(void); +void ptlrpcd_decref(void); + /* ptlrpc/lproc_ptlrpc.c */ #ifdef __KERNEL__ void ptlrpc_lprocfs_register_obd(struct obd_device *obddev); diff --git a/lustre/osc/Makefile.am b/lustre/osc/Makefile.am index 67e8d16..90a5d11 100644 --- a/lustre/osc/Makefile.am +++ b/lustre/osc/Makefile.am @@ -7,12 +7,12 @@ DEFS= if LIBLUSTRE lib_LIBRARIES = libosc.a -libosc_a_SOURCES = osc_request.c osc_lib.c osc_create.c osc_rpcd.c osc_internal.h +libosc_a_SOURCES = osc_request.c osc_lib.c osc_create.c osc_internal.h else MODULE = osc modulefs_DATA = osc.o EXTRA_PROGRAMS = osc -osc_SOURCES = osc_request.c lproc_osc.c osc_lib.c osc_create.c osc_rpcd.c osc_internal.h +osc_SOURCES = osc_request.c lproc_osc.c osc_lib.c osc_create.c osc_internal.h endif include $(top_srcdir)/Rules diff --git a/lustre/osc/osc_create.c b/lustre/osc/osc_create.c index 21a2be7..2f6d431 100644 --- a/lustre/osc/osc_create.c +++ b/lustre/osc/osc_create.c @@ -132,7 +132,7 @@ static int oscc_internal_create(struct osc_creator *oscc) request->rq_async_args.pointer_arg[0] = oscc; request->rq_interpret_reply = osc_interpret_create; - osc_rpcd_add_req(request); + ptlrpcd_add_req(request); RETURN(0); } diff --git a/lustre/osc/osc_internal.h b/lustre/osc/osc_internal.h index a4ed9a8..4b7fa15 100644 --- a/lustre/osc/osc_internal.h +++ b/lustre/osc/osc_internal.h @@ -50,9 +50,6 @@ void oscc_init(struct obd_export *exp); extern atomic_t osc_max_rpcs_in_flight; extern atomic_t osc_max_pages_per_rpc; -int osc_rpcd_addref(void); -int osc_rpcd_decref(void); -void osc_rpcd_add_req(struct ptlrpc_request *req); #ifdef __KERNEL__ diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index 41ca891..8f0d319 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -1343,7 +1343,7 @@ static int osc_send_oap_rpc(struct client_obd *cli, int cmd, page_count, aa, cli->cl_brw_in_flight); request->rq_interpret_reply = brw_interpret_oap; - osc_rpcd_add_req(request); + ptlrpcd_add_req(request); RETURN(1); } @@ -2625,7 +2625,7 @@ static int osc_lock_contains(struct obd_export *exp, struct lov_stripe_md *lsm, } static int osc_invalidate_import(struct obd_device *obd, - struct obd_import *imp) + struct obd_import *imp) { LASSERT(imp->imp_obd == obd); /* this used to try and tear down queued pages, but it was @@ -2639,13 +2639,13 @@ int osc_setup(struct obd_device *obd, obd_count len, void *buf) { int rc; - rc = osc_rpcd_addref(); + rc = ptlrpcd_addref(); if (rc) return rc; rc = client_obd_setup(obd, len, buf); if (rc) - osc_rpcd_decref(); + ptlrpcd_decref(); RETURN(rc); } @@ -2654,7 +2654,7 @@ int osc_cleanup(struct obd_device *obd, int flags) int rc; rc = client_obd_cleanup(obd, flags); - osc_rpcd_decref(); + ptlrpcd_decref(); RETURN(rc); } diff --git a/lustre/osc/osc_rpcd.c b/lustre/osc/osc_rpcd.c deleted file mode 100644 index 603fb0c..0000000 --- a/lustre/osc/osc_rpcd.c +++ /dev/null @@ -1,247 +0,0 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * - * Copyright (C) 2001-2003 Cluster File Systems, Inc. - * Author Peter Braam - * - * This file is part of Lustre, http://www.lustre.org. - * - * Lustre is free software; you can redistribute it and/or - * modify it under the terms of version 2 of the GNU General Public - * License as published by the Free Software Foundation. - * - * Lustre is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Lustre; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - * - * For testing and management it is treated as an obd_device, - * although * it does not export a full OBD method table (the - * requests are coming * in over the wire, so object target modules - * do not have a full * method table.) - * - */ - -#define DEBUG_SUBSYSTEM S_OSC - -#ifdef __KERNEL__ -# include -# include -# include -# include -# include -# if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0)) -# include -# include -# else -# include -# endif -#else /* __KERNEL__ */ -# include -#endif - -#include -#include -#include -#include - -#ifndef __CYGWIN__ -# include -# include -#else -# include -#endif - -#include -#include /* for OBD_FAIL_CHECK */ -#include - -#define LIOD_STOP 0 -static struct osc_rpcd_ctl { - unsigned long orc_flags; - spinlock_t orc_lock; - struct completion orc_starting; - struct completion orc_finishing; - struct list_head orc_req_list; - wait_queue_head_t orc_waitq; - struct ptlrpc_request_set *orc_set; -} osc_orc; - -static DECLARE_MUTEX(osc_rpcd_sem); -static int osc_rpcd_users = 0; - -void osc_rpcd_add_req(struct ptlrpc_request *req) -{ - struct osc_rpcd_ctl *orc = &osc_orc; - - ptlrpc_set_add_new_req(orc->orc_set, req); - wake_up(&orc->orc_waitq); -} - -static int osc_rpcd_check(struct osc_rpcd_ctl *orc) -{ - struct list_head *tmp, *pos; - struct ptlrpc_request *req; - unsigned long flags; - int rc = 0; - ENTRY; - - if (test_bit(LIOD_STOP, &orc->orc_flags)) - RETURN(1); - - spin_lock_irqsave(&orc->orc_set->set_new_req_lock, flags); - list_for_each_safe(pos, tmp, &orc->orc_set->set_new_requests) { - req = list_entry(pos, struct ptlrpc_request, rq_set_chain); - list_del_init(&req->rq_set_chain); - ptlrpc_set_add_req(orc->orc_set, req); - rc = 1; /* need to calculate its timeout */ - } - spin_unlock_irqrestore(&orc->orc_set->set_new_req_lock, flags); - - if (orc->orc_set->set_remaining) { - rc = rc | ptlrpc_check_set(orc->orc_set); - - /* XXX our set never completes, so we prune the completed - * reqs after each iteration. boy could this be smarter. */ - list_for_each_safe(pos, tmp, &orc->orc_set->set_requests) { - req = list_entry(pos, struct ptlrpc_request, - rq_set_chain); - if (req->rq_phase != RQ_PHASE_COMPLETE) - continue; - - list_del_init(&req->rq_set_chain); - req->rq_set = NULL; - ptlrpc_req_finished (req); - } - } - - RETURN(rc); -} - -#ifdef __KERNEL__ -/* ptlrpc's code paths like to execute in process context, so we have this - * thread which spins on a set which contains the io rpcs. llite specifies - * osc_rpcd's set when it pushes pages down into the oscs */ -static int osc_rpcd(void *arg) -{ - struct osc_rpcd_ctl *orc = arg; - unsigned long flags; - ENTRY; - - kportal_daemonize("liod_writeback"); - - SIGNAL_MASK_LOCK(current, flags); - sigfillset(¤t->blocked); - RECALC_SIGPENDING; - SIGNAL_MASK_UNLOCK(current, flags); - - complete(&orc->orc_starting); - - /* like kswapd */ - current->flags |= PF_MEMALLOC; - - /* this mainloop strongly resembles ptlrpc_set_wait except - * that our set never completes. osc_rpcd_check calls ptlrpc_check_set - * when there are requests in the set. new requests come in - * on the set's new_req_list and osc_rpcd_check moves them into - * the set. */ - while (1) { - wait_queue_t set_wait; - struct l_wait_info lwi; - int timeout; - - timeout = ptlrpc_set_next_timeout(orc->orc_set) * HZ; - lwi = LWI_TIMEOUT(timeout, ptlrpc_expired_set, orc->orc_set); - - /* ala the pinger, wait on orc's waitqueue and the set's */ - init_waitqueue_entry(&set_wait, current); - add_wait_queue(&orc->orc_set->set_waitq, &set_wait); - l_wait_event(orc->orc_waitq, osc_rpcd_check(orc), &lwi); - remove_wait_queue(&orc->orc_set->set_waitq, &set_wait); - - if (test_bit(LIOD_STOP, &orc->orc_flags)) - break; - } - /* XXX should be making sure we don't have anything in flight */ - complete(&orc->orc_finishing); - return 0; -} -#else -static int osc_rpcd_recurred = 0; -static void *osc_rpcd_callback; - -int osc_check_async_rpcs(void *arg) -{ - struct osc_rpcd_ctl *orc = arg; - int rc = 0; - - /* single threaded!! */ - osc_rpcd_recurred++; - - if (osc_rpcd_recurred == 1) - rc = osc_rpcd_check(orc); - - osc_rpcd_recurred--; - return rc; -} -#endif - -int osc_rpcd_addref(void) -{ - struct osc_rpcd_ctl *orc = &osc_orc; - int rc = 0; - ENTRY; - - down(&osc_rpcd_sem); - if (++osc_rpcd_users != 1) - GOTO(out, rc); - - memset(orc, 0, sizeof(*orc)); - init_completion(&orc->orc_starting); - init_completion(&orc->orc_finishing); - init_waitqueue_head(&orc->orc_waitq); - orc->orc_flags = 0; - spin_lock_init(&orc->orc_lock); - INIT_LIST_HEAD(&orc->orc_req_list); - - orc->orc_set = ptlrpc_prep_set(); - if (orc->orc_set == NULL) - GOTO(out, rc = -ENOMEM); - -#ifdef __KERNEL__ - if (kernel_thread(osc_rpcd, orc, 0) < 0) { - ptlrpc_set_destroy(orc->orc_set); - GOTO(out, rc = -ECHILD); - } - - wait_for_completion(&orc->orc_starting); -#else - osc_rpcd_callback = - liblustre_register_wait_callback(&osc_check_async_rpcs, orc); -#endif -out: - up(&osc_rpcd_sem); - RETURN(rc); -} - -void osc_rpcd_decref(void) -{ - struct osc_rpcd_ctl *orc = &osc_orc; - - down(&osc_rpcd_sem); - if (--osc_rpcd_users == 0) { - set_bit(LIOD_STOP, &orc->orc_flags); - wake_up(&orc->orc_waitq); -#ifdef __KERNEL__ - wait_for_completion(&orc->orc_finishing); -#else - liblustre_deregister_wait_callback(osc_rpcd_callback); -#endif - ptlrpc_set_destroy(orc->orc_set); - } - up(&osc_rpcd_sem); -} diff --git a/lustre/ptlrpc/Makefile.am b/lustre/ptlrpc/Makefile.am index 09f935f..2c6de45 100644 --- a/lustre/ptlrpc/Makefile.am +++ b/lustre/ptlrpc/Makefile.am @@ -11,12 +11,14 @@ LDLMSOURCES= $(top_srcdir)/ldlm/l_lock.c $(top_srcdir)/ldlm/ldlm_lock.c \ $(top_srcdir)/ldlm/ldlm_flock.c $(top_srcdir)/ldlm/ldlm_request.c \ $(top_srcdir)/ldlm/ldlm_lockd.c $(top_srcdir)/ldlm/ldlm_internal.h +COMMON_SOURCES = client.c recover.c connection.c niobuf.c pack_generic.c \ + events.c ptlrpc_module.c service.c pinger.c recov_thread.c llog_net.c \ + llog_client.c import.c ptlrpcd.c $(LDLMSOURCES) + if LIBLUSTRE lib_LIBRARIES = libptlrpc.a -libptlrpc_a_SOURCES = client.c niobuf.c pack_generic.c recover.c connection.c \ -ptlrpc_module.c events.c service.c pinger.c import.c \ -recov_thread.c llog_net.c llog_client.c $(LDLMSOURCES) +libptlrpc_a_SOURCES = $(COMMON_SOURCES) else @@ -24,10 +26,8 @@ MODULE = ptlrpc modulefs_DATA = ptlrpc.o EXTRA_PROGRAMS = ptlrpc -ptlrpc_SOURCES = recover.c connection.c ptlrpc_module.c events.c service.c \ - client.c niobuf.c pack_generic.c lproc_ptlrpc.c pinger.c \ - ptlrpc_internal.h recov_thread.c import.c llog_net.c llog_server.c \ - llog_client.c $(LDLMSOURCES) +ptlrpc_SOURCES = $(COMMON_SOURCES) lproc_ptlrpc.c ptlrpc_internal.h \ + llog_server.c endif ptlrpc_DEPENDENCIES=symlinks diff --git a/lustre/ptlrpc/ptlrpc_module.c b/lustre/ptlrpc/ptlrpc_module.c index 44e3ebc..1268c37 100644 --- a/lustre/ptlrpc/ptlrpc_module.c +++ b/lustre/ptlrpc/ptlrpc_module.c @@ -188,6 +188,11 @@ EXPORT_SYMBOL(ptlrpc_pinger_add_import); EXPORT_SYMBOL(ptlrpc_pinger_del_import); EXPORT_SYMBOL(ptlrpc_pinger_sending_on_import); +/* ptlrpcd.c */ +EXPORT_SYMBOL(ptlrpcd_addref); +EXPORT_SYMBOL(ptlrpcd_decref); +EXPORT_SYMBOL(ptlrpcd_add_req); + /* lproc_ptlrpc.c */ EXPORT_SYMBOL(ptlrpc_lprocfs_register_obd); EXPORT_SYMBOL(ptlrpc_lprocfs_unregister_obd); diff --git a/lustre/ptlrpc/ptlrpcd.c b/lustre/ptlrpc/ptlrpcd.c new file mode 100644 index 0000000..b7b9700 --- /dev/null +++ b/lustre/ptlrpc/ptlrpcd.c @@ -0,0 +1,240 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001-2003 Cluster File Systems, Inc. + * Author Peter Braam + * + * This file is part of Lustre, http://www.lustre.org. + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#define DEBUG_SUBSYSTEM S_RPC + +#ifdef __KERNEL__ +# include +# include +# include +# include +# include +# if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0)) +# include +# include +# else +# include +# endif +#else /* __KERNEL__ */ +# include +#endif + +#include +#include + +#ifndef __CYGWIN__ +# include +# include +#else +# include +#endif + +#include +#include /* for OBD_FAIL_CHECK */ +#include + +#define LIOD_STOP 0 +static struct ptlrpcd_ctl { + unsigned long pc_flags; + spinlock_t pc_lock; + struct completion pc_starting; + struct completion pc_finishing; + struct list_head pc_req_list; + wait_queue_head_t pc_waitq; + struct ptlrpc_request_set *pc_set; +} ptlrpcd_pc; + +static DECLARE_MUTEX(ptlrpcd_sem); +static int ptlrpcd_users = 0; + +void ptlrpcd_add_req(struct ptlrpc_request *req) +{ + struct ptlrpcd_ctl *pc = &ptlrpcd_pc; + + ptlrpc_set_add_new_req(pc->pc_set, req); + wake_up(&pc->pc_waitq); +} + +static int ptlrpcd_check(struct ptlrpcd_ctl *pc) +{ + struct list_head *tmp, *pos; + struct ptlrpc_request *req; + unsigned long flags; + int rc = 0; + ENTRY; + + if (test_bit(LIOD_STOP, &pc->pc_flags)) + RETURN(1); + + spin_lock_irqsave(&pc->pc_set->set_new_req_lock, flags); + list_for_each_safe(pos, tmp, &pc->pc_set->set_new_requests) { + req = list_entry(pos, struct ptlrpc_request, rq_set_chain); + list_del_init(&req->rq_set_chain); + ptlrpc_set_add_req(pc->pc_set, req); + rc = 1; /* need to calculate its timeout */ + } + spin_unlock_irqrestore(&pc->pc_set->set_new_req_lock, flags); + + if (pc->pc_set->set_remaining) { + rc = rc | ptlrpc_check_set(pc->pc_set); + + /* XXX our set never completes, so we prune the completed + * reqs after each iteration. boy could this be smarter. */ + list_for_each_safe(pos, tmp, &pc->pc_set->set_requests) { + req = list_entry(pos, struct ptlrpc_request, + rq_set_chain); + if (req->rq_phase != RQ_PHASE_COMPLETE) + continue; + + list_del_init(&req->rq_set_chain); + req->rq_set = NULL; + ptlrpc_req_finished (req); + } + } + + RETURN(rc); +} + +#ifdef __KERNEL__ +/* ptlrpc's code paths like to execute in process context, so we have this + * thread which spins on a set which contains the io rpcs. llite specifies + * ptlrpcd's set when it pushes pages down into the oscs */ +static int ptlrpcd(void *arg) +{ + struct ptlrpcd_ctl *pc = arg; + unsigned long flags; + ENTRY; + + kportal_daemonize("ptlrpcd"); + + SIGNAL_MASK_LOCK(current, flags); + sigfillset(¤t->blocked); + RECALC_SIGPENDING; + SIGNAL_MASK_UNLOCK(current, flags); + + complete(&pc->pc_starting); + + /* like kswapd */ + current->flags |= PF_MEMALLOC; + + /* this mainloop strongly resembles ptlrpc_set_wait except + * that our set never completes. ptlrpcd_check calls ptlrpc_check_set + * when there are requests in the set. new requests come in + * on the set's new_req_list and ptlrpcd_check moves them into + * the set. */ + while (1) { + wait_queue_t set_wait; + struct l_wait_info lwi; + int timeout; + + timeout = ptlrpc_set_next_timeout(pc->pc_set) * HZ; + lwi = LWI_TIMEOUT(timeout, ptlrpc_expired_set, pc->pc_set); + + /* ala the pinger, wait on pc's waitqueue and the set's */ + init_waitqueue_entry(&set_wait, current); + add_wait_queue(&pc->pc_set->set_waitq, &set_wait); + l_wait_event(pc->pc_waitq, ptlrpcd_check(pc), &lwi); + remove_wait_queue(&pc->pc_set->set_waitq, &set_wait); + + if (test_bit(LIOD_STOP, &pc->pc_flags)) + break; + } + /* XXX should be making sure we don't have anything in flight */ + complete(&pc->pc_finishing); + return 0; +} +#else +static int ptlrpcd_recurred = 0; +static void *ptlrpcd_callback; + +int ptlrpcd_check_async_rpcs(void *arg) +{ + struct ptlrpcd_ctl *pc = arg; + int rc = 0; + + /* single threaded!! */ + ptlrpcd_recurred++; + + if (ptlrpcd_recurred == 1) + rc = ptlrpcd_check(pc); + + ptlrpcd_recurred--; + return rc; +} +#endif + +int ptlrpcd_addref(void) +{ + struct ptlrpcd_ctl *pc = &ptlrpcd_pc; + int rc = 0; + ENTRY; + + down(&ptlrpcd_sem); + if (++ptlrpcd_users != 1) + GOTO(out, rc); + + memset(pc, 0, sizeof(*pc)); + init_completion(&pc->pc_starting); + init_completion(&pc->pc_finishing); + init_waitqueue_head(&pc->pc_waitq); + pc->pc_flags = 0; + spin_lock_init(&pc->pc_lock); + INIT_LIST_HEAD(&pc->pc_req_list); + + pc->pc_set = ptlrpc_prep_set(); + if (pc->pc_set == NULL) + GOTO(out, rc = -ENOMEM); + +#ifdef __KERNEL__ + if (kernel_thread(ptlrpcd, pc, 0) < 0) { + ptlrpc_set_destroy(pc->pc_set); + GOTO(out, rc = -ECHILD); + } + + wait_for_completion(&pc->pc_starting); +#else + ptlrpcd_callback = + liblustre_register_wait_callback(&ptlrpcd_check_async_rpcs, pc); +#endif +out: + up(&ptlrpcd_sem); + RETURN(rc); +} + +void ptlrpcd_decref(void) +{ + struct ptlrpcd_ctl *pc = &ptlrpcd_pc; + + down(&ptlrpcd_sem); + if (--ptlrpcd_users == 0) { + set_bit(LIOD_STOP, &pc->pc_flags); + wake_up(&pc->pc_waitq); +#ifdef __KERNEL__ + wait_for_completion(&pc->pc_finishing); +#else + liblustre_deregister_wait_callback(ptlrpcd_callback); +#endif + ptlrpc_set_destroy(pc->pc_set); + } + up(&ptlrpcd_sem); +} -- 1.8.3.1