X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Fptlrpc%2Fimport.c;h=2ea79c249e35b0e54a41cb4507a61015e1351a2c;hb=b2f4fd040713bea3ad3d3f2540048c0ee8e07def;hp=a85421452ca2d2a0b54e6af40b0adff75d94aa47;hpb=f4c8cde0bc7dd683a1746e6aaf3f9f29443a9229;p=fs%2Flustre-release.git diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c index a854214..2ea79c2 100644 --- a/lustre/ptlrpc/import.c +++ b/lustre/ptlrpc/import.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include "ptlrpc_internal.h" @@ -163,15 +164,12 @@ int ptlrpc_set_import_discon(struct obd_import *imp, __u32 conn_cnt) return rc; } -/* - * This acts as a barrier; all existing requests are rejected, and - * no new requests will be accepted until the import is valid again. - */ -void ptlrpc_deactivate_import(struct obd_import *imp) +/* Must be called with imp_lock held! */ +static void ptlrpc_deactivate_and_unlock_import(struct obd_import *imp) { ENTRY; + LASSERT_SPIN_LOCKED(&imp->imp_lock); - spin_lock(&imp->imp_lock); if (imp->imp_invalid) { spin_unlock(&imp->imp_lock); EXIT; @@ -190,6 +188,16 @@ void ptlrpc_deactivate_import(struct obd_import *imp) } /* + * This acts as a barrier; all existing requests are rejected, and + * no new requests will be accepted until the import is valid again. + */ +void ptlrpc_deactivate_import(struct obd_import *imp) +{ + spin_lock(&imp->imp_lock); + ptlrpc_deactivate_and_unlock_import(imp); +} + +/* * This function will invalidate the import, if necessary, then block * for all the RPC completions, and finally notify the obd to * invalidate its state (ie cancel locks, clear pending requests, @@ -202,7 +210,15 @@ void ptlrpc_invalidate_import(struct obd_import *imp) atomic_inc(&imp->imp_inval_count); - ptlrpc_deactivate_import(imp); + /* + * If this is an invalid MGC connection, then don't bother + * waiting for imp_inflight to drop to 0. + */ + if (imp->imp_invalid && imp->imp_recon_bk && !imp->imp_obd->obd_no_recov) + goto out; + + if (!imp->imp_invalid || imp->imp_obd->obd_no_recov) + ptlrpc_deactivate_import(imp); LASSERT(imp->imp_invalid); @@ -212,11 +228,26 @@ void ptlrpc_invalidate_import(struct obd_import *imp) rc = l_wait_event(imp->imp_recovery_waitq, (atomic_read(&imp->imp_inflight) == 0), &lwi); - if (rc) - CDEBUG(D_HA, "%s: rc = %d waiting for callback (%d != 0)\n", + if (rc) { + struct list_head *tmp, *n; + struct ptlrpc_request *req; + + CERROR("%s: rc = %d waiting for callback (%d != 0)\n", obd2cli_tgt(imp->imp_obd), rc, atomic_read(&imp->imp_inflight)); + spin_lock(&imp->imp_lock); + list_for_each_safe(tmp, n, &imp->imp_sending_list) { + req = list_entry(tmp, struct ptlrpc_request, rq_list); + DEBUG_REQ(D_ERROR, req, "still on sending list"); + } + list_for_each_safe(tmp, n, &imp->imp_delayed_list) { + req = list_entry(tmp, struct ptlrpc_request, rq_list); + DEBUG_REQ(D_ERROR, req, "still on delayed list"); + } + spin_unlock(&imp->imp_lock); + } +out: obd_import_event(imp->imp_obd, imp, IMP_EVENT_INVALIDATE); sptlrpc_import_flush_all_ctx(imp); @@ -264,6 +295,37 @@ void ptlrpc_fail_import(struct obd_import *imp, __u32 conn_cnt) EXIT; } +int ptlrpc_reconnect_import(struct obd_import *imp) +{ + + ptlrpc_set_import_discon(imp, 0); + /* Force a new connect attempt */ + ptlrpc_invalidate_import(imp); + /* Do a fresh connect next time by zeroing the handle */ + ptlrpc_disconnect_import(imp, 1); + /* Wait for all invalidate calls to finish */ + if (atomic_read(&imp->imp_inval_count) > 0) { + int rc; + struct l_wait_info lwi = LWI_INTR(LWI_ON_SIGNAL_NOOP, NULL); + rc = l_wait_event(imp->imp_recovery_waitq, + (atomic_read(&imp->imp_inval_count) == 0), + &lwi); + if (rc) + CERROR("Interrupted, inval=%d\n", + atomic_read(&imp->imp_inval_count)); + } + + /* Allow reconnect attempts */ + imp->imp_obd->obd_no_recov = 0; + /* Remove 'invalid' flag */ + ptlrpc_activate_import(imp); + /* Attempt a new connect */ + ptlrpc_recover_import(imp, NULL); + return 0; +} + +EXPORT_SYMBOL(ptlrpc_reconnect_import); + static int import_select_connection(struct obd_import *imp) { struct obd_import_conn *imp_conn = NULL, *conn; @@ -353,12 +415,17 @@ int ptlrpc_first_transno(struct obd_import *imp, __u64 *transno) { struct ptlrpc_request *req; struct list_head *tmp; - + if (list_empty(&imp->imp_replay_list)) return 0; tmp = imp->imp_replay_list.next; req = list_entry(tmp, struct ptlrpc_request, rq_replay_list); *transno = req->rq_transno; + if (req->rq_transno == 0) { + DEBUG_REQ(D_ERROR, req, "zero transno in replay"); + LBUG(); + } + return 1; } @@ -367,20 +434,15 @@ int ptlrpc_connect_import(struct obd_import *imp, char *new_uuid) struct obd_device *obd = imp->imp_obd; int initial_connect = 0; int set_transno = 0; - int rc; __u64 committed_before_reconnect = 0; struct ptlrpc_request *request; - int size[] = { sizeof(struct ptlrpc_body), - sizeof(imp->imp_obd->u.cli.cl_target_uuid), - sizeof(obd->obd_uuid), - sizeof(imp->imp_dlm_handle), - sizeof(imp->imp_connect_data) }; - char *tmp[] = { NULL, - obd2cli_tgt(imp->imp_obd), - obd->obd_uuid.uuid, - (char *)&imp->imp_dlm_handle, - (char *)&imp->imp_connect_data }; + char *bufs[] = { NULL, + obd2cli_tgt(imp->imp_obd), + obd->obd_uuid.uuid, + (char *)&imp->imp_dlm_handle, + (char *)&imp->imp_connect_data }; struct ptlrpc_connect_async_args *aa; + int rc; ENTRY; spin_lock(&imp->imp_lock); @@ -445,19 +507,29 @@ int ptlrpc_connect_import(struct obd_import *imp, char *new_uuid) } } + rc = sptlrpc_import_sec_adapt(imp, NULL, 0); + if (rc) + GOTO(out, rc); + /* Reset connect flags to the originally requested flags, in case * the server is updated on-the-fly we will get the new features. */ imp->imp_connect_data.ocd_connect_flags = imp->imp_connect_flags_orig; - rc = obd_reconnect(imp->imp_obd->obd_self_export, obd, + rc = obd_reconnect(NULL, imp->imp_obd->obd_self_export, obd, &obd->obd_uuid, &imp->imp_connect_data); if (rc) GOTO(out, rc); - request = ptlrpc_prep_req(imp, LUSTRE_OBD_VERSION, imp->imp_connect_op, - 5, size, tmp); - if (!request) + request = ptlrpc_request_alloc(imp, &RQF_MDS_CONNECT); + if (request == NULL) GOTO(out, rc = -ENOMEM); + rc = ptlrpc_request_bufs_pack(request, LUSTRE_OBD_VERSION, + imp->imp_connect_op, bufs, NULL); + if (rc) { + ptlrpc_request_free(request); + GOTO(out, rc); + } + #ifndef __KERNEL__ lustre_msg_add_op_flags(request->rq_reqmsg, MSG_CONNECT_LIBCLIENT); #endif @@ -465,9 +537,9 @@ int ptlrpc_connect_import(struct obd_import *imp, char *new_uuid) request->rq_send_state = LUSTRE_IMP_CONNECTING; /* Allow a slightly larger reply for future growth compatibility */ - size[REPLY_REC_OFF] = sizeof(struct obd_connect_data) + - 16 * sizeof(__u64); - ptlrpc_req_set_repsize(request, 2, size); + req_capsule_set_size(&request->rq_pill, &RMF_CONNECT_DATA, RCL_SERVER, + sizeof(struct obd_connect_data)+16*sizeof(__u64)); + ptlrpc_request_set_replen(request); request->rq_interpret_reply = ptlrpc_connect_interpret; CLASSERT(sizeof (*aa) <= sizeof (request->rq_async_args)); @@ -526,7 +598,12 @@ static void ptlrpc_maybe_ping_import_soon(struct obd_import *imp) struct obd_import_conn, oic_item); - if (imp->imp_conn_current != imp_conn) { + /* XXX: When the failover node is the primary node, it is possible + * to have two identical connections in imp_conn_list. We must + * compare not conn's pointers but NIDs, otherwise we can defeat + * connection throttling. (See bug 14774.) */ + if (imp->imp_conn_current->oic_conn->c_self != + imp_conn->oic_conn->c_self) { ptlrpc_ping_import_soon(imp); wake_pinger = 1; } @@ -583,20 +660,21 @@ static int ptlrpc_connect_interpret(struct ptlrpc_request *request, spin_unlock(&imp->imp_lock); } - if (msg_flags & MSG_CONNECT_NEXT_VER) { - imp->imp_msg_magic = LUSTRE_MSG_MAGIC_V2; - CDEBUG(D_RPCTRACE, "connect to %s with lustre_msg_v2\n", - obd2cli_tgt(imp->imp_obd)); - } else { - CDEBUG(D_RPCTRACE, "connect to %s with lustre_msg_v1\n", - obd2cli_tgt(imp->imp_obd)); - } + /* if applies, adjust the imp->imp_msg_magic here + * according to reply flags */ imp->imp_remote_handle = *lustre_msg_get_handle(request->rq_repmsg); IMPORT_SET_STATE(imp, LUSTRE_IMP_FULL); - ptlrpc_activate_import(imp); + spin_lock(&imp->imp_lock); + if (imp->imp_invalid) { + spin_unlock(&imp->imp_lock); + ptlrpc_activate_import(imp); + } else { + spin_unlock(&imp->imp_lock); + } + GOTO(finish, rc = 0); } else { spin_unlock(&imp->imp_lock); @@ -617,22 +695,19 @@ static int ptlrpc_connect_interpret(struct ptlrpc_request *request, if (memcmp(&imp->imp_remote_handle, lustre_msg_get_handle(request->rq_repmsg), sizeof(imp->imp_remote_handle))) { - int level = D_ERROR; - /* Old MGC can reconnect to a restarted MGS */ - if (strcmp(imp->imp_obd->obd_type->typ_name, - LUSTRE_MGC_NAME) == 0) { - level = D_CONFIG; - } - CDEBUG(level, - "%s@%s changed handle from "LPX64" to "LPX64 - "; copying, but this may foreshadow disaster\n", + + CWARN("%s@%s changed server handle from " + LPX64" to "LPX64" - evicting.\n", obd2cli_tgt(imp->imp_obd), imp->imp_connection->c_remote_uuid.uuid, imp->imp_remote_handle.cookie, lustre_msg_get_handle(request->rq_repmsg)-> - cookie); + cookie); imp->imp_remote_handle = *lustre_msg_get_handle(request->rq_repmsg); + + IMPORT_SET_STATE(imp, LUSTRE_IMP_EVICTED); + GOTO(finish, rc = 0); } else { CDEBUG(D_HA, "reconnected to %s@%s after partition\n", obd2cli_tgt(imp->imp_obd), @@ -696,9 +771,12 @@ finish: } else { struct obd_connect_data *ocd; struct obd_export *exp; - - ocd = lustre_swab_repbuf(request, REPLY_REC_OFF, sizeof(*ocd), - lustre_swab_connect); + int ret; + ret = req_capsule_get_size(&request->rq_pill, &RMF_CONNECT_DATA, + RCL_SERVER); + /* server replied obd_connect_data is always bigger */ + ocd = req_capsule_server_sized_get(&request->rq_pill, + &RMF_CONNECT_DATA, ret); spin_lock(&imp->imp_lock); list_del(&imp->imp_conn_current->oic_item); @@ -770,6 +848,38 @@ finish: newer : older, LUSTRE_VERSION_STRING); } + if (ocd->ocd_connect_flags & OBD_CONNECT_CKSUM) { + /* We sent to the server ocd_cksum_types with bits set + * for algorithms we understand. The server masked off + * the checksum types it doesn't support */ + if ((ocd->ocd_cksum_types & OBD_CKSUM_ALL) == 0) { + LCONSOLE_WARN("The negotiation of the checksum " + "alogrithm to use with server %s " + "failed (%x/%x), disabling " + "checksums\n", + obd2cli_tgt(imp->imp_obd), + ocd->ocd_cksum_types, + OBD_CKSUM_ALL); + cli->cl_checksum = 0; + cli->cl_supp_cksum_types = OBD_CKSUM_CRC32; + cli->cl_cksum_type = OBD_CKSUM_CRC32; + } else { + cli->cl_supp_cksum_types = ocd->ocd_cksum_types; + + if (ocd->ocd_cksum_types & OSC_DEFAULT_CKSUM) + cli->cl_cksum_type = OSC_DEFAULT_CKSUM; + else if (ocd->ocd_cksum_types & OBD_CKSUM_ADLER) + cli->cl_cksum_type = OBD_CKSUM_ADLER; + else + cli->cl_cksum_type = OBD_CKSUM_CRC32; + } + } else { + /* The server does not support OBD_CONNECT_CKSUM. + * Enforce CRC32 for backward compatibility*/ + cli->cl_supp_cksum_types = OBD_CKSUM_CRC32; + cli->cl_cksum_type = OBD_CKSUM_CRC32; + } + if (ocd->ocd_connect_flags & OBD_CONNECT_BRW_SIZE) { cli->cl_max_pages_per_rpc = ocd->ocd_brw_size >> CFS_PAGE_SHIFT; @@ -787,9 +897,12 @@ finish: out: if (rc != 0) { IMPORT_SET_STATE(imp, LUSTRE_IMP_DISCON); + spin_lock(&imp->imp_lock); if (aa->pcaa_initial_connect && !imp->imp_initial_recov && (request->rq_import_generation == imp->imp_generation)) - ptlrpc_deactivate_import(imp); + ptlrpc_deactivate_and_unlock_import(imp); + else + spin_unlock(&imp->imp_lock); if ((imp->imp_recon_bk && imp->imp_last_recon) || (rc == -EACCES)) { @@ -805,12 +918,11 @@ out: struct obd_connect_data *ocd; /* reply message might not be ready */ - if (request->rq_repmsg != NULL) + if (request->rq_repmsg == NULL) RETURN(-EPROTO); - ocd = lustre_swab_repbuf(request, REPLY_REC_OFF, - sizeof *ocd, - lustre_swab_connect); + ocd = req_capsule_server_get(&request->rq_pill, + &RMF_CONNECT_DATA); if (ocd && (ocd->ocd_connect_flags & OBD_CONNECT_VERSION) && (ocd->ocd_version != LUSTRE_VERSION_CODE)) { @@ -874,13 +986,14 @@ static int signal_completed_replay(struct obd_import *imp) LASSERT(atomic_read(&imp->imp_replay_inflight) == 0); atomic_inc(&imp->imp_replay_inflight); - req = ptlrpc_prep_req(imp, LUSTRE_OBD_VERSION, OBD_PING, 1, NULL, NULL); - if (!req) { + req = ptlrpc_request_alloc_pack(imp, &RQF_OBD_PING, LUSTRE_OBD_VERSION, + OBD_PING); + if (req == NULL) { atomic_dec(&imp->imp_replay_inflight); RETURN(-ENOMEM); } - ptlrpc_req_set_repsize(req, 1, NULL); + ptlrpc_request_set_replen(req); req->rq_send_state = LUSTRE_IMP_REPLAY_WAIT; lustre_msg_add_flags(req->rq_reqmsg, MSG_LOCK_REPLAY_DONE | MSG_REQ_REPLAY_DONE); @@ -1062,7 +1175,8 @@ int ptlrpc_disconnect_import(struct obd_import *imp, int noclose) spin_unlock(&imp->imp_lock); - req = ptlrpc_prep_req(imp, LUSTRE_OBD_VERSION, rq_opc, 1, NULL, NULL); + req = ptlrpc_request_alloc_pack(imp, &RQF_MDS_DISCONNECT, + LUSTRE_OBD_VERSION, rq_opc); if (req) { /* We are disconnecting, do not retry a failed DISCONNECT rpc if * it fails. We can get through the above with a down server @@ -1075,7 +1189,7 @@ int ptlrpc_disconnect_import(struct obd_import *imp, int noclose) #endif IMPORT_SET_STATE(imp, LUSTRE_IMP_CONNECTING); req->rq_send_state = LUSTRE_IMP_CONNECTING; - ptlrpc_req_set_repsize(req, 1, NULL); + ptlrpc_request_set_replen(req); rc = ptlrpc_queue_wait(req); ptlrpc_req_finished(req); }