From: shaver Date: Fri, 4 Oct 2002 23:07:07 +0000 (+0000) Subject: - Add support for umount -f: it invalidates all in-flight and delayed requests X-Git-Tag: 0.5.14~58 X-Git-Url: https://git.whamcloud.com/gitweb?a=commitdiff_plain;h=b29c41b491ba6c4e3575b4765fb6a20589d7399a;p=fs%2Flustre-release.git - Add support for umount -f: it invalidates all in-flight and delayed requests and marks the connection(s) as invalid, so that future operations fail. - Make the NEWCONN ioctl clear the CONN_INVALID flag. - Remove a handful of unused members from ptlrpc_request. - More informative and uniform req-failure (timeout, recovery, umount -f) reporting. - Update runfailure-net a little; more work needed here. - Add --force / -f to lconf for "umount -f" umounting. --- diff --git a/lustre/include/linux/lustre_net.h b/lustre/include/linux/lustre_net.h index 3422642..92b5712 100644 --- a/lustre/include/linux/lustre_net.h +++ b/lustre/include/linux/lustre_net.h @@ -68,6 +68,8 @@ #define OST_MAXREQSIZE (8 * 1024) #endif +#define CONN_INVALID 1 + struct ptlrpc_connection { struct list_head c_link; struct lustre_peer c_peer; @@ -98,6 +100,7 @@ struct ptlrpc_connection { struct list_head c_imports; struct list_head c_exports; struct list_head c_sb_chain; + __u32 c_flags; /* can we indicate INVALID elsewhere? */ }; struct ptlrpc_client { @@ -130,11 +133,9 @@ struct ptlrpc_client { struct ptlrpc_request { int rq_type; /* one of PTL_RPC_MSG_* */ struct list_head rq_list; - struct list_head rq_multi; struct obd_device *rq_obd; int rq_status; int rq_flags; - __u32 rq_connid; atomic_t rq_refcount; int rq_reqlen; @@ -145,23 +146,18 @@ struct ptlrpc_request { __u64 rq_transno; __u64 rq_xid; - char *rq_bulkbuf; - int rq_bulklen; - int rq_level; - time_t rq_time; time_t rq_timeout; // void * rq_reply_handle; wait_queue_head_t rq_wait_for_rep; /* incoming reply */ ptl_md_t rq_reply_md; - ptl_handle_md_t rq_reply_md_h; + ptl_handle_md_t rq_reply_md_h; /* we can lose this: set, never read */ ptl_handle_me_t rq_reply_me_h; /* outgoing req/rep */ ptl_md_t rq_req_md; - ptl_handle_md_t rq_req_md_h; struct lustre_peer rq_peer; /* XXX see service.c can this be factored away? */ struct obd_export *rq_export; diff --git a/lustre/llite/super.c b/lustre/llite/super.c index 05c6876..35629c4 100644 --- a/lustre/llite/super.c +++ b/lustre/llite/super.c @@ -524,6 +524,42 @@ static void ll_read_inode2(struct inode *inode, void *opaque) } } +static inline void invalidate_request_list(struct list_head *req_list) +{ + struct list_head *tmp, *n; + list_for_each_safe(tmp, n, req_list) { + struct ptlrpc_request *req = + list_entry(tmp, struct ptlrpc_request, rq_list); + CERROR("invalidating req xid %d op %d to %s:%d\n", + (unsigned long long)req->rq_xid, req->rq_reqmsg->opc, + req->rq_connection->c_remote_uuid, + req->rq_import->imp_client->cli_request_portal); + req->rq_flags |= PTL_RPC_FL_ERR; + wake_up(&req->rq_wait_for_rep); + } +} + +void ll_umount_begin(struct super_block *sb) +{ + struct ll_sb_info *sbi = ll_s2sbi(sb); + struct list_head *ctmp; + + ENTRY; + + list_for_each(ctmp, &sbi->ll_conn_chain) { + struct ptlrpc_connection *conn; + conn = list_entry(ctmp, struct ptlrpc_connection, c_sb_chain); + + spin_lock(&conn->c_lock); + conn->c_flags |= CONN_INVALID; + invalidate_request_list(&conn->c_sending_head); + invalidate_request_list(&conn->c_delayed_head); + spin_unlock(&conn->c_unlock); + } + + EXIT; +} + /* exported operations */ struct super_operations ll_super_operations = { @@ -531,7 +567,8 @@ struct super_operations ll_super_operations = clear_inode: ll_clear_inode, delete_inode: ll_delete_inode, put_super: ll_put_super, - statfs: ll_statfs + statfs: ll_statfs, + umount_begin: ll_umount_begin }; struct file_system_type lustre_lite_fs_type = { diff --git a/lustre/ptlrpc/client.c b/lustre/ptlrpc/client.c index 1097b58..a09af9e 100644 --- a/lustre/ptlrpc/client.c +++ b/lustre/ptlrpc/client.c @@ -172,7 +172,6 @@ struct ptlrpc_request *ptlrpc_prep_req(struct obd_import *imp, int opcode, request->rq_connection = ptlrpc_connection_addref(conn); INIT_LIST_HEAD(&request->rq_list); - INIT_LIST_HEAD(&request->rq_multi); /* * This will be reduced once when the sender is finished (waiting for * reply, f.e.), once when the request has been committed and is @@ -231,7 +230,6 @@ void ptlrpc_free_req(struct ptlrpc_request *request) } ptlrpc_put_connection(request->rq_connection); - list_del(&request->rq_multi); OBD_FREE(request, sizeof(*request)); EXIT; } @@ -447,10 +445,10 @@ static int expired_request(void *data) struct ptlrpc_request *req = data; ENTRY; - CERROR("req timeout on connid %d xid %Ld portal %d op %d\n", - req->rq_connid, (unsigned long long)req->rq_xid, - req->rq_import->imp_client->cli_request_portal, - req->rq_reqmsg->opc); + CERROR("req xid "LPD64" op %d: timeout on conn to %s:%d\n", + (unsigned long long)req->rq_xid, req->rq_reqmsg->opc, + req->rq_connection->c_remote_uuid, + req->rq_import->imp_client->cli_request_portal); req->rq_flags |= PTL_RPC_FL_TIMEOUT; if (!req->rq_import->imp_connection->c_recovd_data.rd_recovd) RETURN(1); @@ -490,32 +488,57 @@ int ptlrpc_queue_wait(struct ptlrpc_request *req) /* XXX probably both an import and connection level are needed */ if (req->rq_level > conn->c_level) { - CERROR("pid %d waiting for recovery (%d > %d) on conn %p(%s)\n", - current->pid, req->rq_level, conn->c_level, conn, - conn->c_remote_uuid); - spin_lock(&conn->c_lock); + if (conn->c_flags & CONN_INVALID) { + /* being torn down by "umount -f" */ + CERROR("req xid "LPD64" op %d to %s:%d: CONN_INVALID\n", + (unsigned long long)req->rq_xid, + req->rq_reqmsg->opc, + req->rq_connection->c_remote_uuid, + req->rq_import->imp_client->cli_request_portal); + spin_unlock(&conn->c_lock); + RETURN(-EIO); + } list_del(&req->rq_list); list_add_tail(&req->rq_list, &conn->c_delayed_head); spin_unlock(&conn->c_lock); + CERROR("req xid "LPD64" op %d to %s:%d: waiting for recovery " + "(%d < %d)\n", + (unsigned long long)req->rq_xid, req->rq_reqmsg->opc, + req->rq_connection->c_remote_uuid, + req->rq_import->imp_client->cli_request_portal, + req->rq_level, conn->c_level); + lwi = LWI_INTR(NULL, NULL); rc = l_wait_event(req->rq_wait_for_rep, - req->rq_level <= conn->c_level, &lwi); + (req->rq_level <= conn->c_level) || + (req->rq_flags & PTL_RPC_FL_ERR), &lwi); spin_lock(&conn->c_lock); list_del_init(&req->rq_list); spin_unlock(&conn->c_lock); + if (req->rq_flags & PTL_RPC_FL_ERR) + RETURN(-EIO); + if (rc) RETURN(rc); - + CERROR("process %d resumed\n", current->pid); } resend: - req->rq_time = CURRENT_TIME; req->rq_timeout = obd_timeout; spin_lock(&conn->c_lock); + if (conn->c_flags & CONN_INVALID) { + CERROR("req xid "LPD64" op %d to %s:%d: CONN_INVALID\n", + (unsigned long long)req->rq_xid, req->rq_reqmsg->opc, + req->rq_connection->c_remote_uuid, + req->rq_import->imp_client->cli_request_portal); + spin_unlock(&conn->c_lock); /* being torn down by "umount -f" */ + RETURN(-EIO); + } + list_del(&req->rq_list); list_add_tail(&req->rq_list, &conn->c_sending_head); spin_unlock(&conn->c_lock); @@ -599,7 +622,6 @@ int ptlrpc_replay_req(struct ptlrpc_request *req) req->rq_xid, req->rq_reqmsg->opc, req->rq_level, req->rq_connection->c_level); - req->rq_time = CURRENT_TIME; req->rq_timeout = obd_timeout; req->rq_reqmsg->addr = req->rq_import->imp_handle.addr; req->rq_reqmsg->cookie = req->rq_import->imp_handle.cookie; diff --git a/lustre/ptlrpc/connection.c b/lustre/ptlrpc/connection.c index 07b46eb..e643013 100644 --- a/lustre/ptlrpc/connection.c +++ b/lustre/ptlrpc/connection.c @@ -84,6 +84,7 @@ struct ptlrpc_connection *ptlrpc_get_connection(struct lustre_peer *peer, c->c_generation = 1; c->c_epoch = 1; c->c_bootcount = 0; + c->c_flags = 0; if (uuid) strcpy(c->c_remote_uuid, uuid); INIT_LIST_HEAD(&c->c_delayed_head); diff --git a/lustre/ptlrpc/rpc.c b/lustre/ptlrpc/rpc.c index ceefc33..7b28ffa 100644 --- a/lustre/ptlrpc/rpc.c +++ b/lustre/ptlrpc/rpc.c @@ -120,11 +120,21 @@ int connmgr_iocontrol(long cmd, struct lustre_handle *hdl, int len, void *karg, goto out; } + /* else (NEWCONN) */ - if (conn->c_recovd_data.rd_phase != RD_PREPARING) + spin_lock(&conn->c_lock); + + /* whatever happens, reset the INVALID flag */ + conn->c_flags &= ~CONN_INVALID; + + /* XXX is this a good check? should we allow readdressing of + * XXX conns that aren't in recovery? + */ + if (conn->c_recovd_data.rd_phase != RD_PREPARING) { + spin_unlock(&conn->c_lock); GOTO(out, rc = -EALREADY); + } - spin_lock(&conn->c_lock); if (data->ioc_inllen2) { CERROR("conn %p UUID change %s -> %s\n", conn, conn->c_remote_uuid, data->ioc_inlbuf2); diff --git a/lustre/tests/runfailure-net b/lustre/tests/runfailure-net index 078e1a4..ce5634b 100755 --- a/lustre/tests/runfailure-net +++ b/lustre/tests/runfailure-net @@ -1,28 +1,39 @@ #!/bin/sh -set -vx - -SRCDIR="`dirname $0`" -. $SRCDIR/common.sh +fail() { + echo "ERROR: $1" 1>&2 + [ $2 ] && RC=$2 || RC=1 + exit $RC +} test_fail() { + oldtimeout=`cat /proc/sys/lustre/timeout` + echo $TIMEOUT > /proc/sys/lustre/timeout echo $1 > /proc/sys/lustre/fail_loc shift $* & - sleep 1 + sleep $TIMEOUT + sleep 2 # fudge kill -9 $! + echo $oldtimeout > /proc/sys/lustre/timeout echo 0 > /proc/sys/lustre/fail_loc - umount /mnt/lustre || fail "cannot unmount /mnt/lustre" - mount -t lustre_lite -o device=`$OBDCTL name2dev OSCDEV` none /mnt/lustre || fail "cannot remount device '`$OBDCTL name2dev OSCDEV`' on /mnt/lustre" + umount -f /mnt/lustre || fail "cannot unmount /mnt/lustre" + mount -t lustre_lite -o "osc=$OSC,mdc=$MDC" none /mnt/lustre || \ + fail "cannot remount $OSC/$MDC on /mnt/lustre" } -[ -c /dev/request ] || mknod /dev/request c 10 244 +set -vx + +LCTL=../utils/lctl +OSC=OSC_localhost_UUID +MDC=MDC_client1_UUID +TIMEOUT=5 # complete in finite time [ "`mount | grep /mnt/lustre`" ] || echo | sh llmount.sh || exit -1 # GETATTR_NET - ls will hang on the getattr -test_fail 0x102 ls -l /mnt/lustre +# test_fail 0x102 ls -l /mnt/lustre # READPAGE_NET - ls will hang reading in new pages (lost+found is not in cache) test_fail 0x104 ls /mnt/lustre diff --git a/lustre/utils/lconf b/lustre/utils/lconf index 467cc0f..7bdb9e8 100755 --- a/lustre/utils/lconf +++ b/lustre/utils/lconf @@ -52,6 +52,7 @@ config.xml Lustre configuration in xml format. --get URL to fetch a config file --node Load config for -d | --cleanup Cleans up config. (Shutdown) +-f | --force Unmount with \"umount -f\" during shutdown -v | --verbose Print system commands as they are run -h | --help Print this help --gdb Prints message after creating gdb module script @@ -86,6 +87,7 @@ class Config: self._gdb = 0 self._nomod = 0 self._nosetup = 0 + self._force = 0 # parameters self._modules = None self._node = None @@ -123,6 +125,10 @@ class Config: if flag: self._nosetup = flag return self._nosetup + def force(self, flag = None): + if flag: self._force = flag + return self._flag + def node(self, val = None): if val: self._node = val return self._node @@ -1053,7 +1059,10 @@ class Mountpoint(Module): def cleanup(self): self.info(self.path, self.mds_uuid,self.lov_uuid) - (rc, out) = run("umount", self.path) + if config.force(): + (rc, out) = run("umount -f", self.path) + else: + (rc, out) = run("umount", self.path) if rc: log("umount failed, cleanup will most likely not work.") l = lookup(self.dom_node.parentNode, self.lov_uuid) @@ -1474,18 +1483,20 @@ def parse_cmdline(argv): config.portals = a if o == "--lustre": config.lustre = a - if o == "--reformat": + if o == "--reformat": config.reformat(1) - if o == "--node": + if o == "--node": config.node(a) - if o == "--gdb": + if o == "--gdb": config.gdb(1) - if o == "--nomod": + if o == "--nomod": config.nomod(1) - if o == "--nosetup": + if o == "--nosetup": config.nosetup(1) - if o == "--dump": + if o == "--dump": config.dump_file(a) + if o in ("--force", "-f"): + config.force(1) return args def fetch(url):