From f083146f15af3d98c2c910b966d2291097dc868f Mon Sep 17 00:00:00 2001 From: green Date: Mon, 16 Feb 2004 10:19:41 +0000 Subject: [PATCH] Update to HEAD. Should now contain fix for 2710/2743 --- lustre/liblustre/Makefile.am | 69 ++++++-------- lustre/ptlrpc/pinger.c | 209 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 225 insertions(+), 53 deletions(-) diff --git a/lustre/liblustre/Makefile.am b/lustre/liblustre/Makefile.am index ef4fa2f..6622485 100644 --- a/lustre/liblustre/Makefile.am +++ b/lustre/liblustre/Makefile.am @@ -1,63 +1,46 @@ ## Liblustre excecutables & libraries Makefile DEFS= +SUBDIRS = . tests + CFLAGS := -g -Wall -I$(top_srcdir)/utils -I$(top_srcdir)/portals/include \ - -I$(top_srcdir)/portals/unals -I$(SYSIO)/include \ - -I/opt/lam/include -L/opt/lam/lib + -I$(top_srcdir)/portals/unals -I$(SYSIO)/include -KFLAGS:= CPPFLAGS = $(HAVE_EFENCE) -D_LARGEFILE64_SOURCE=1 LIBS = $(LIBEFENCE) -## lustre components libs -LLIBS := ./libllite.a \ - ../lov/liblov.a \ - ../obdecho/libobdecho.a \ - ../osc/libosc.a \ - ../mdc/libmdc.a \ - ../ldlm/libldlm.a \ - ../ptlrpc/libptlrpc.a \ - ../obdclass/liblustreclass.a \ - ../lvfs/liblvfs.a - -## portals components libs -PTLLIBS := ../portals/utils/libptlctl.a \ - ../portals/unals/libtcpnal.a \ - ../portals/portals/libportals.a - -## sysio components libs -SYSIOLIBS := $(SYSIO)/drivers/native/libsysio_native.a \ +LUSTRE_LIBS = libllite.a \ + $(top_srcdir)/lov/liblov.a \ + $(top_srcdir)/obdecho/libobdecho.a \ + $(top_srcdir)/osc/libosc.a \ + $(top_srcdir)/mdc/libmdc.a \ + $(top_srcdir)/ptlrpc/libptlrpc.a \ + $(top_srcdir)/obdclass/liblustreclass.a \ + $(top_srcdir)/lvfs/liblvfs.a + +PTL_LIBS = $(top_srcdir)/portals/utils/libuptlctl.a \ + $(top_srcdir)/portals/unals/libtcpnal.a \ + $(top_srcdir)/portals/portals/libportals.a + +SYSIO_LIBS = $(SYSIO)/drivers/native/libsysio_native.a \ $(SYSIO)/drivers/sockets/libsysio_sockets.a \ $(SYSIO)/src/libsysio.a \ $(SYSIO)/dev/stdfd/libsysio_stdfd.a -LLIB_EXEC= $(PTLLIBS) $(SYSIOLIBS) -lpthread +#SYSIO_LIBS = $(SYSIO)/lib/libsysio.a -lib_LIBRARIES = -noinst_LIBRARIES = libllite.a libtestcommon.a -libllite_a_SOURCES = llite_lib.c super.c namei.c rw.c file.c -libtestcommon_a_SOURCES = test_common.c +lib_LIBRARIES = liblustre.a +noinst_LIBRARIES = libllite.a -bin_PROGRAMS = libtest lltest recovery_small replay_single #test_lock_cancel +libllite_a_SOURCES = llite_lib.c super.c namei.c rw.c file.c dir.c +libllite_a_CFLAGS = -fPIC -libtest_SOURCES = libtest.c ../utils/parser.c ../utils/obd.c ../utils/lustre_cfg.c -libtest_LDADD := $(LLIBS) $(PTLLIBS) \ - $(LIBREADLINE) -lpthread +# for make rpms -- need cleanup +liblustre_a_SOURCES = llite_lib.c super.c namei.c rw.c file.c dir.c +liblustre_a_CFLAGS = -fPIC -liblustre.a : libllite.a +liblustre.a : $(LUSTRE_LIBS) $(PTL_LIBS) $(SYSIO_LIBS) $(shell ./genlib.sh $(SYSIO) $(AR) $(LINK)) -lltest_SOURCES = lltest.c -lltest_LDADD := ./libtestcommon.a $(LLIBS) $(LLIB_EXEC) $(LIBREADLINE) - -recovery_small_SOURCES = recovery_small.c -recovery_small_LDADD := ./libtestcommon.a $(LLIBS) $(LLIB_EXEC) $(LIBREADLINE) - -replay_single_SOURCES = replay_single.c -replay_single_LDADD := ./libtestcommon.a $(LLIBS) $(LLIB_EXEC) $(LIBREADLINE) - -#test_lock_cancel_SOURCES = test_lock_cancel.c -#test_lock_cancel_LDADD := $(LLIBS) $(LLIB_EXEC) -lmpi -llam - include $(top_srcdir)/Rules diff --git a/lustre/ptlrpc/pinger.c b/lustre/ptlrpc/pinger.c index 3caf74e..ab85900 100644 --- a/lustre/ptlrpc/pinger.c +++ b/lustre/ptlrpc/pinger.c @@ -35,12 +35,12 @@ #include #include "ptlrpc_internal.h" -#ifdef __KERNEL__ - -static struct ptlrpc_thread *pinger_thread = NULL; static DECLARE_MUTEX(pinger_sem); static struct list_head pinger_imports = LIST_HEAD_INIT(pinger_imports); +#ifdef __KERNEL__ +static struct ptlrpc_thread *pinger_thread = NULL; + static int ptlrpc_pinger_main(void *arg) { struct ptlrpc_svc_data *data = (struct ptlrpc_svc_data *)arg; @@ -307,30 +307,219 @@ int ptlrpc_pinger_del_import(struct obd_import *imp) RETURN(0); } -#else /* !__KERNEL__ */ +#else +/* XXX + * the current implementation of pinger in liblustre is not optimized + */ + +static struct pinger_data { + int pd_recursion; + unsigned long pd_this_ping; + unsigned long pd_next_ping; + struct ptlrpc_request_set *pd_set; +} pinger_args; + +static int pinger_check_rpcs(void *arg) +{ + unsigned long curtime = time(NULL); + struct ptlrpc_request *req; + struct ptlrpc_request_set *set; + struct list_head *iter; + struct pinger_data *pd = &pinger_args; + int rc; + + /* prevent recursion */ + if (pd->pd_recursion++) { + CDEBUG(D_HA, "pinger: recursion! quit\n"); + LASSERT(pd->pd_set); + pd->pd_recursion--; + return 0; + } + + /* have we reached ping point? */ + if (!pd->pd_set && pd->pd_next_ping > curtime) { + pd->pd_recursion--; + return 0; + } + + /* if we have rpc_set already, continue processing it */ + if (pd->pd_set) { + LASSERT(pd->pd_this_ping); + set = pd->pd_set; + goto do_check_set; + } + + pd->pd_this_ping = curtime; + pd->pd_set = ptlrpc_prep_set(); + set = pd->pd_set; + + /* add rpcs into set */ + down(&pinger_sem); + list_for_each(iter, &pinger_imports) { + struct obd_import *imp = + list_entry(iter, struct obd_import, + imp_pinger_chain); + int generation, level; + unsigned long flags; + + if (imp->imp_next_ping <= pd->pd_this_ping) { + /* Add a ping. */ + spin_lock_irqsave(&imp->imp_lock, flags); + generation = imp->imp_generation; + level = imp->imp_state; + spin_unlock_irqrestore(&imp->imp_lock, flags); + + if (level != LUSTRE_IMP_FULL) { + CDEBUG(D_HA, + "not pinging %s (in recovery)\n", + imp->imp_target_uuid.uuid); + continue; + } + + req = ptlrpc_prep_req(imp, OBD_PING, 0, NULL, + NULL); + if (!req) { + CERROR("out of memory\n"); + break; + } + req->rq_no_resend = 1; + req->rq_replen = lustre_msg_size(0, NULL); + req->rq_send_state = LUSTRE_IMP_FULL; + req->rq_phase = RQ_PHASE_RPC; + req->rq_import_generation = generation; + ptlrpc_set_add_req(set, req); + } else { + CDEBUG(D_HA, "don't need to ping %s (%lu > " + "%lu)\n", imp->imp_target_uuid.uuid, + imp->imp_next_ping, pd->pd_this_ping); + } + } + pd->pd_this_ping = curtime; + up(&pinger_sem); + + /* Might be empty, that's OK. */ + if (set->set_remaining == 0) + CDEBUG(D_HA, "nothing to ping\n"); + + list_for_each(iter, &set->set_requests) { + struct ptlrpc_request *req = + list_entry(iter, struct ptlrpc_request, + rq_set_chain); + DEBUG_REQ(D_HA, req, "pinging %s->%s", + req->rq_import->imp_obd->obd_uuid.uuid, + req->rq_import->imp_target_uuid.uuid); + (void)ptl_send_rpc(req); + } + +do_check_set: + rc = ptlrpc_check_set(set); + + /* not finished, and we are not expired, simply return */ + if (!rc && curtime < pd->pd_this_ping + obd_timeout) { + CDEBUG(D_HA, "not finished, but also not expired\n"); + pd->pd_recursion--; + return 0; + } + + /* Expire all the requests that didn't come back. */ + down(&pinger_sem); + list_for_each(iter, &set->set_requests) { + req = list_entry(iter, struct ptlrpc_request, + rq_set_chain); + + if (req->rq_replied) + continue; + + req->rq_phase = RQ_PHASE_COMPLETE; + set->set_remaining--; + /* If it was disconnected, don't sweat it. */ + if (list_empty(&req->rq_import->imp_pinger_chain)) { + ptlrpc_unregister_reply(req); + continue; + } + + CDEBUG(D_HA, "pinger initiate expire_one_request\n"); + ptlrpc_expire_one_request(req); + } + up(&pinger_sem); + + ptlrpc_set_destroy(set); + pd->pd_set = NULL; + + pd->pd_next_ping = pd->pd_this_ping + obd_timeout; + pd->pd_this_ping = 0; /* XXX for debug */ + + CDEBUG(D_HA, "finished a round ping\n"); + pd->pd_recursion--; + return 0; +} + +static void *pinger_callback = NULL; int ptlrpc_start_pinger(void) { + memset(&pinger_args, 0, sizeof(pinger_args)); +#ifdef ENABLE_PINGER + pinger_callback = + liblustre_register_wait_callback(&pinger_check_rpcs, &pinger_args); +#endif + obd_timeout = 10; return 0; } int ptlrpc_stop_pinger(void) { +#ifdef ENABLE_PINGER + if (pinger_callback) + liblustre_deregister_wait_callback(pinger_callback); +#endif return 0; } -int ptlrpc_pinger_add_import(struct obd_import *imp) +void ptlrpc_pinger_sending_on_import(struct obd_import *imp) { - return 0; + down(&pinger_sem); + imp->imp_next_ping = time(NULL) + obd_timeout; + if (pinger_args.pd_set == NULL && + pinger_args.pd_next_ping > imp->imp_next_ping) { + CDEBUG(D_HA, "set next ping to %ld(cur %ld)\n", + imp->imp_next_ping, time(NULL)); + pinger_args.pd_next_ping = imp->imp_next_ping; + } + up(&pinger_sem); } -int ptlrpc_pinger_del_import(struct obd_import *imp) +int ptlrpc_pinger_add_import(struct obd_import *imp) { - return 0; + ENTRY; + if (!list_empty(&imp->imp_pinger_chain)) + RETURN(-EALREADY); + + CDEBUG(D_HA, "adding pingable import %s->%s\n", + imp->imp_obd->obd_uuid.uuid, imp->imp_target_uuid.uuid); + ptlrpc_pinger_sending_on_import(imp); + + down(&pinger_sem); + list_add_tail(&imp->imp_pinger_chain, &pinger_imports); + class_import_get(imp); + up(&pinger_sem); + + RETURN(0); } -void ptlrpc_pinger_sending_on_import(struct obd_import *imp) +int ptlrpc_pinger_del_import(struct obd_import *imp) { + ENTRY; + if (list_empty(&imp->imp_pinger_chain)) + RETURN(-ENOENT); + + down(&pinger_sem); + list_del_init(&imp->imp_pinger_chain); + CDEBUG(D_HA, "removing pingable import %s->%s\n", + imp->imp_obd->obd_uuid.uuid, imp->imp_target_uuid.uuid); + class_import_put(imp); + up(&pinger_sem); + RETURN(0); } -#endif +#endif /* !__KERNEL__ */ -- 1.8.3.1