Whamcloud - gitweb
LU-631 osc: wait OSC to complete initial connection with OST
authorHongchao Zhang <hongchao.zhang@intel.com>
Mon, 8 Apr 2013 02:49:09 +0000 (10:49 +0800)
committerOleg Drokin <oleg.drokin@intel.com>
Thu, 25 Apr 2013 17:12:13 +0000 (13:12 -0400)
in functions lov_prep_*_set, if one OSC isn't active and it is trying
its first connection with OST, then spend some time (obd_timeout)
to wait it to complete.

Change-Id: I424dbf81b6ceebf2cfd1cf48b0f89be40c4c3df4
Signed-off-by: Hongchao Zhang <hongchao.zhang@intel.com>
Reviewed-on: http://review.whamcloud.com/2469
Tested-by: Hudson
Reviewed-by: Fan Yong <fan.yong@intel.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Jeremy Filizetti <jeremy.filizetti@gmail.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
lustre/include/lustre_import.h
lustre/lov/lov_internal.h
lustre/lov/lov_request.c
lustre/ptlrpc/import.c

index cb1c04b..98c2c92 100644 (file)
@@ -250,23 +250,41 @@ struct obd_import {
         /** Protects flags, level, generation, conn_cnt, *_list */
        spinlock_t                imp_lock;
 
-        /* flags */
-        unsigned long             imp_no_timeout:1,       /* timeouts are disabled */
-                                  imp_invalid:1,          /* evicted */
-                                  imp_deactive:1,         /* administratively disabled */
-                                  imp_replayable:1,       /* try to recover the import */
-                                  imp_dlm_fake:1,         /* don't run recovery (timeout instead) */
-                                  imp_server_timeout:1,   /* use 1/2 timeout on MDS' OSCs */
-                                  imp_delayed_recovery:1, /* VBR: imp in delayed recovery */
-                                  imp_no_lock_replay:1,   /* VBR: if gap was found then no lock replays */
-                                  imp_vbr_failed:1,       /* recovery by versions was failed */
-                                  imp_force_verify:1,     /* force an immidiate ping */
-                                 imp_force_next_verify:1,/* force a scheduled ping */
-                                  imp_pingable:1,         /* pingable */
-                                  imp_resend_replay:1,    /* resend for replay */
-                                  imp_no_pinger_recover:1,/* disable normal recovery, for test only. */
-                                 imp_need_mne_swab:1,    /* need IR MNE swab */
-                                  imp_force_reconnect:1;  /* import must be reconnected instead of chouse new connection */
+       /* flags */
+       unsigned long             imp_no_timeout:1, /* timeouts are disabled */
+                                 imp_invalid:1,    /* evicted */
+                                 /* administratively disabled */
+                                 imp_deactive:1,
+                                 /* try to recover the import */
+                                 imp_replayable:1,
+                                 /* don't run recovery (timeout instead) */
+                                 imp_dlm_fake:1,
+                                 /* use 1/2 timeout on MDS' OSCs */
+                                 imp_server_timeout:1,
+                                 /* VBR: imp in delayed recovery */
+                                 imp_delayed_recovery:1,
+                                 /* VBR: if gap was found then no lock replays
+                                  */
+                                 imp_no_lock_replay:1,
+                                 /* recovery by versions was failed */
+                                 imp_vbr_failed:1,
+                                 /* force an immidiate ping */
+                                 imp_force_verify:1,
+                                 /* force a scheduled ping */
+                                 imp_force_next_verify:1,
+                                 /* pingable */
+                                 imp_pingable:1,
+                                 /* resend for replay */
+                                 imp_resend_replay:1,
+                                 /* disable normal recovery, for test only. */
+                                 imp_no_pinger_recover:1,
+                                 /* need IR MNE swab */
+                                 imp_need_mne_swab:1,
+                                 /* import must be reconnected instead of
+                                  * chouse new connection */
+                                 imp_force_reconnect:1,
+                                 /* import has tried to connect with server */
+                                 imp_connect_tried:1;
         __u32                     imp_connect_op;
         struct obd_connect_data   imp_connect_data;
         __u64                     imp_connect_flags_orig;
index 7332696..de959ae 100644 (file)
@@ -174,6 +174,7 @@ void lov_update_set(struct lov_request_set *set,
                     struct lov_request *req, int rc);
 int lov_update_common_set(struct lov_request_set *set,
                           struct lov_request *req, int rc);
+int lov_check_and_wait_active(struct lov_obd *lov, int ost_idx);
 int lov_prep_create_set(struct obd_export *exp, struct obd_info *oifo,
                         struct lov_stripe_md **ea, struct obdo *src_oa,
                         struct obd_trans_info *oti,
index c6d8e84..e6c5181 100644 (file)
@@ -146,6 +146,62 @@ void lov_set_add_req(struct lov_request *req, struct lov_request_set *set)
         req->rq_rqset = set;
 }
 
+static int lov_check_set(struct lov_obd *lov, int idx)
+{
+       int rc = 0;
+       mutex_lock(&lov->lov_lock);
+
+       if (lov->lov_tgts[idx] == NULL ||
+           lov->lov_tgts[idx]->ltd_active ||
+           (lov->lov_tgts[idx]->ltd_exp != NULL &&
+            class_exp2cliimp(lov->lov_tgts[idx]->ltd_exp)->imp_connect_tried))
+               rc = 1;
+
+       mutex_unlock(&lov->lov_lock);
+       return rc;
+}
+
+/* Check if the OSC connection exists and is active.
+ * If the OSC has not yet had a chance to connect to the OST the first time,
+ * wait once for it to connect instead of returning an error.
+ */
+int lov_check_and_wait_active(struct lov_obd *lov, int ost_idx)
+{
+       cfs_waitq_t waitq;
+       struct l_wait_info lwi;
+       struct lov_tgt_desc *tgt;
+       int rc = 0;
+
+       mutex_lock(&lov->lov_lock);
+
+       tgt = lov->lov_tgts[ost_idx];
+
+       if (unlikely(tgt == NULL))
+               GOTO(out, rc = 0);
+
+       if (likely(tgt->ltd_active))
+               GOTO(out, rc = 1);
+
+       if (tgt->ltd_exp && class_exp2cliimp(tgt->ltd_exp)->imp_connect_tried)
+               GOTO(out, rc = 0);
+
+       mutex_unlock(&lov->lov_lock);
+
+       cfs_waitq_init(&waitq);
+       lwi = LWI_TIMEOUT_INTERVAL(cfs_time_seconds(obd_timeout),
+                                  cfs_time_seconds(1), NULL, NULL);
+
+       rc = l_wait_event(waitq, lov_check_set(lov, ost_idx), &lwi);
+       if (tgt != NULL && tgt->ltd_active)
+               return 1;
+
+       return 0;
+
+out:
+       mutex_unlock(&lov->lov_lock);
+       return rc;
+}
+
 extern void osc_update_enqueue(struct lustre_handle *lov_lockhp,
                                struct lov_oinfo *loi, int flags,
                                struct ost_lvb *lvb, __u32 mode, int rc);
@@ -344,11 +400,10 @@ int lov_prep_enqueue_set(struct obd_export *exp, struct obd_info *oinfo,
                                            &start, &end))
                         continue;
 
-                if (!lov->lov_tgts[loi->loi_ost_idx] ||
-                    !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) {
-                        CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
-                        continue;
-                }
+               if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) {
+                       CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
+                       continue;
+               }
 
                 OBD_ALLOC(req, sizeof(*req));
                 if (req == NULL)
@@ -452,11 +507,10 @@ int lov_prep_match_set(struct obd_export *exp, struct obd_info *oinfo,
                         continue;
 
                 /* FIXME raid1 should grace this error */
-                if (!lov->lov_tgts[loi->loi_ost_idx] ||
-                    !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) {
-                        CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
-                        GOTO(out_set, rc = -EIO);
-                }
+               if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) {
+                       CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
+                       GOTO(out_set, rc = -EIO);
+               }
 
                 OBD_ALLOC(req, sizeof(*req));
                 if (req == NULL)
@@ -712,11 +766,10 @@ int lov_prep_brw_set(struct obd_export *exp, struct obd_info *oinfo,
                         continue;
 
                 loi = oinfo->oi_md->lsm_oinfo[i];
-                if (!lov->lov_tgts[loi->loi_ost_idx] ||
-                    !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) {
-                        CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
-                        GOTO(out, rc = -EIO);
-                }
+               if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) {
+                       CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
+                       GOTO(out, rc = -EIO);
+               }
 
                 OBD_ALLOC(req, sizeof(*req));
                 if (req == NULL)
@@ -829,18 +882,17 @@ int lov_prep_getattr_set(struct obd_export *exp, struct obd_info *oinfo,
         set->set_oi = oinfo;
 
         for (i = 0; i < oinfo->oi_md->lsm_stripe_count; i++) {
-                struct lov_oinfo *loi;
-                struct lov_request *req;
-
-                loi = oinfo->oi_md->lsm_oinfo[i];
-                if (!lov->lov_tgts[loi->loi_ost_idx] ||
-                    !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) {
-                        CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
-                        if (oinfo->oi_oa->o_valid & OBD_MD_FLEPOCH)
-                                /* SOM requires all the OSTs to be active. */
-                                GOTO(out_set, rc = -EIO);
-                        continue;
-                }
+               struct lov_oinfo *loi;
+               struct lov_request *req;
+
+               loi = oinfo->oi_md->lsm_oinfo[i];
+               if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) {
+                       CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
+                       if (oinfo->oi_oa->o_valid & OBD_MD_FLEPOCH)
+                               /* SOM requires all the OSTs to be active. */
+                               GOTO(out_set, rc = -EIO);
+                       continue;
+               }
 
                 OBD_ALLOC(req, sizeof(*req));
                 if (req == NULL)
@@ -911,15 +963,14 @@ int lov_prep_destroy_set(struct obd_export *exp, struct obd_info *oinfo,
                 set->set_cookies = oti->oti_logcookies;
 
         for (i = 0; i < lsm->lsm_stripe_count; i++) {
-                struct lov_oinfo *loi;
-                struct lov_request *req;
+               struct lov_oinfo *loi;
+               struct lov_request *req;
 
-                loi = lsm->lsm_oinfo[i];
-                if (!lov->lov_tgts[loi->loi_ost_idx] ||
-                    !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) {
-                        CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
-                        continue;
-                }
+               loi = lsm->lsm_oinfo[i];
+               if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) {
+                       CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
+                       continue;
+               }
 
                 OBD_ALLOC(req, sizeof(*req));
                 if (req == NULL)
@@ -1023,14 +1074,13 @@ int lov_prep_setattr_set(struct obd_export *exp, struct obd_info *oinfo,
                 set->set_cookies = oti->oti_logcookies;
 
         for (i = 0; i < oinfo->oi_md->lsm_stripe_count; i++) {
-                struct lov_oinfo *loi = oinfo->oi_md->lsm_oinfo[i];
-                struct lov_request *req;
+               struct lov_oinfo *loi = oinfo->oi_md->lsm_oinfo[i];
+               struct lov_request *req;
 
-                if (!lov->lov_tgts[loi->loi_ost_idx] ||
-                    !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) {
-                        CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
-                        continue;
-                }
+               if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) {
+                       CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
+                       continue;
+               }
 
                 OBD_ALLOC(req, sizeof(*req));
                 if (req == NULL)
@@ -1157,11 +1207,10 @@ int lov_prep_punch_set(struct obd_export *exp, struct obd_info *oinfo,
                                            &rs, &re))
                         continue;
 
-                if (!lov->lov_tgts[loi->loi_ost_idx] ||
-                    !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) {
-                        CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
-                        GOTO(out_set, rc = -EIO);
-                }
+               if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) {
+                       CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
+                       GOTO(out_set, rc = -EIO);
+               }
 
                 OBD_ALLOC(req, sizeof(*req));
                 if (req == NULL)
@@ -1247,15 +1296,14 @@ int lov_prep_sync_set(struct obd_export *exp, struct obd_info *oinfo,
         set->set_oi = oinfo;
 
         for (i = 0; i < oinfo->oi_md->lsm_stripe_count; i++) {
-                struct lov_oinfo *loi = oinfo->oi_md->lsm_oinfo[i];
-                struct lov_request *req;
-                obd_off rs, re;
+               struct lov_oinfo *loi = oinfo->oi_md->lsm_oinfo[i];
+               struct lov_request *req;
+               obd_off rs, re;
 
-                if (!lov->lov_tgts[loi->loi_ost_idx] ||
-                    !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) {
-                        CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
-                        continue;
-                }
+               if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) {
+                       CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
+                       continue;
+               }
 
                 if (!lov_stripe_intersects(oinfo->oi_md, i, start, end, &rs,
                                            &re))
@@ -1478,13 +1526,14 @@ int lov_prep_statfs_set(struct obd_device *obd, struct obd_info *oinfo,
 
         /* We only get block data from the OBD */
         for (i = 0; i < lov->desc.ld_tgt_count; i++) {
-                struct lov_request *req;
-
-                if (!lov->lov_tgts[i] || (!lov->lov_tgts[i]->ltd_active
-                                          && (oinfo->oi_flags & OBD_STATFS_NODELAY))) {
-                        CDEBUG(D_HA, "lov idx %d inactive\n", i);
-                        continue;
-                }
+               struct lov_request *req;
+
+               if (lov->lov_tgts[i] == NULL ||
+                   (!lov_check_and_wait_active(lov, i) &&
+                    (oinfo->oi_flags & OBD_STATFS_NODELAY))) {
+                       CDEBUG(D_HA, "lov idx %d inactive\n", i);
+                       continue;
+               }
 
                 /* skip targets that have been explicitely disabled by the
                  * administrator */
index 7eaee96..cdb91cf 100644 (file)
@@ -771,6 +771,7 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
 
        spin_lock(&imp->imp_lock);
        if (imp->imp_state == LUSTRE_IMP_CLOSED) {
+               imp->imp_connect_tried = 1;
                spin_unlock(&imp->imp_lock);
                RETURN(0);
        }
@@ -985,17 +986,18 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
         }
 
 finish:
-        rc = ptlrpc_import_recovery_state_machine(imp);
-        if (rc != 0) {
-                if (rc == -ENOTCONN) {
-                        CDEBUG(D_HA, "evicted/aborted by %s@%s during recovery;"
-                               "invalidating and reconnecting\n",
-                               obd2cli_tgt(imp->imp_obd),
-                               imp->imp_connection->c_remote_uuid.uuid);
-                        ptlrpc_connect_import(imp);
-                        RETURN(0);
-                }
-        } else {
+       rc = ptlrpc_import_recovery_state_machine(imp);
+       if (rc != 0) {
+               if (rc == -ENOTCONN) {
+                       CDEBUG(D_HA, "evicted/aborted by %s@%s during recovery;"
+                              "invalidating and reconnecting\n",
+                              obd2cli_tgt(imp->imp_obd),
+                              imp->imp_connection->c_remote_uuid.uuid);
+                       ptlrpc_connect_import(imp);
+                       imp->imp_connect_tried = 1;
+                       RETURN(0);
+               }
+       } else {
 
                spin_lock(&imp->imp_lock);
                cfs_list_del(&imp->imp_conn_current->oic_item);
@@ -1133,6 +1135,8 @@ finish:
         }
 
 out:
+       imp->imp_connect_tried = 1;
+
         if (rc != 0) {
                 IMPORT_SET_STATE(imp, LUSTRE_IMP_DISCON);
                 if (rc == -EACCES) {