Whamcloud - gitweb
LU-248 increase LOV reseed to mitigate OST allocation inconsitence
[fs/lustre-release.git] / lustre / lov / lov_qos.c
index 59f20c3..c1f8c96 100644 (file)
@@ -26,7 +26,7 @@
  * GPL HEADER END
  */
 /*
- * Copyright  2008 Sun Microsystems, Inc. All rights reserved
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Use is subject to license terms.
  */
 /*
@@ -461,7 +461,7 @@ void qos_shrink_lsm(struct lov_request_set *set)
                 oti_alloc_cookies(set->set_oti, set->set_count);
                 if (set->set_oti->oti_logcookies) {
                         memcpy(set->set_oti->oti_logcookies, cookies, newsize);
-                        OBD_FREE(cookies, oldsize);
+                        OBD_FREE_LARGE(cookies, oldsize);
                         set->set_cookies = set->set_oti->oti_logcookies;
                 } else {
                         CWARN("'leaking' %d bytes\n", oldsize - newsize);
@@ -473,7 +473,7 @@ void qos_shrink_lsm(struct lov_request_set *set)
         LASSERT(lsm->lsm_stripe_count >= set->set_count);
 
         newsize = lov_stripe_md_size(set->set_count);
-        OBD_ALLOC(lsm_new, newsize);
+        OBD_ALLOC_LARGE(lsm_new, newsize);
         if (lsm_new != NULL) {
                 int i;
                 memcpy(lsm_new, lsm, sizeof(*lsm));
@@ -486,45 +486,88 @@ void qos_shrink_lsm(struct lov_request_set *set)
                                       sizeof(struct lov_oinfo));
                 }
                 lsm_new->lsm_stripe_count = set->set_count;
-                OBD_FREE(lsm, sizeof(struct lov_stripe_md) +
-                         lsm->lsm_stripe_count * sizeof(struct lov_oinfo *));
+                OBD_FREE_LARGE(lsm, sizeof(struct lov_stripe_md) +
+                               lsm->lsm_stripe_count*sizeof(struct lov_oinfo*));
                 set->set_oi->oi_md = lsm_new;
         } else {
                 CWARN("'leaking' few bytes\n");
         }
 }
 
+/**
+ * Check whether we can create the object on the OST(refered by ost_idx)
+ * \retval:
+ *          0: create the object.
+ *          other value: did not create the object.
+ */
+static int lov_check_and_create_object(struct lov_obd *lov, int ost_idx,
+                                       struct lov_stripe_md *lsm,
+                                       struct lov_request *req,
+                                       struct obd_trans_info *oti)
+{
+        int stripe;
+        int rc = -EIO;
+        ENTRY;
+
+        CDEBUG(D_QOS, "Check and create on idx %d \n", ost_idx);
+        if (!lov->lov_tgts[ost_idx] ||
+            !lov->lov_tgts[ost_idx]->ltd_active)
+                RETURN(rc);
+
+        /* check if objects has been created on this ost */
+        for (stripe = 0; stripe < lsm->lsm_stripe_count; stripe++) {
+                /* already have object at this stripe */
+                if (ost_idx == lsm->lsm_oinfo[stripe]->loi_ost_idx)
+                        break;
+        }
+
+        if (stripe >= lsm->lsm_stripe_count) {
+                req->rq_idx = ost_idx;
+                rc = obd_create(lov->lov_tgts[ost_idx]->ltd_exp,
+                                req->rq_oi.oi_oa, &req->rq_oi.oi_md,
+                                oti);
+        }
+        RETURN(rc);
+}
+
 int qos_remedy_create(struct lov_request_set *set, struct lov_request *req)
 {
         struct lov_stripe_md *lsm = set->set_oi->oi_md;
         struct lov_obd *lov = &set->set_exp->exp_obd->u.lov;
-        unsigned ost_idx, ost_count = lov->desc.ld_tgt_count;
-        int stripe, i, rc = -EIO;
+        unsigned ost_idx = 0, ost_count;
+        struct pool_desc *pool;
+        struct ost_pool *osts = NULL;
+        int i, rc = -EIO;
         ENTRY;
 
-        ost_idx = (req->rq_idx + lsm->lsm_stripe_count) % ost_count;
-        for (i = 0; i < ost_count; i++, ost_idx = (ost_idx + 1) % ost_count) {
-                if (!lov->lov_tgts[ost_idx] ||
-                    !lov->lov_tgts[ost_idx]->ltd_active)
-                        continue;
-                /* check if objects has been created on this ost */
-                for (stripe = 0; stripe < lsm->lsm_stripe_count; stripe++) {
-                        /* we try send create to this ost but he is failed */
-                        if (stripe == req->rq_stripe)
-                                continue;
-                        /* already have object at this stripe */
-                        if (ost_idx == lsm->lsm_oinfo[stripe]->loi_ost_idx)
-                                break;
-                }
-                if (stripe >= lsm->lsm_stripe_count) {
-                        req->rq_idx = ost_idx;
-                        rc = obd_create(lov->lov_tgts[ost_idx]->ltd_exp,
-                                        req->rq_oi.oi_oa, &req->rq_oi.oi_md,
-                                        set->set_oti);
-                        if (!rc)
+        /* First check whether we can create the objects on the pool */
+        pool = lov_find_pool(lov, lsm->lsm_pool_name);
+        if (pool != NULL) {
+                cfs_down_read(&pool_tgt_rw_sem(pool));
+                osts = &(pool->pool_obds);
+                ost_count = osts->op_count;
+                for (i = 0; i < ost_count; i++, ost_idx = osts->op_array[i]) {
+                        rc = lov_check_and_create_object(lov, ost_idx, lsm, req,
+                                                         set->set_oti);
+                        if (rc == 0)
                                 break;
                 }
+                cfs_up_read(&pool_tgt_rw_sem(pool));
+                lov_pool_putref(pool);
+                RETURN(rc);
         }
+
+        ost_count = lov->desc.ld_tgt_count;
+        /* Then check whether we can create the objects on other OSTs */
+        ost_idx = (req->rq_idx + lsm->lsm_stripe_count) % ost_count;
+        for (i = 0; i < ost_count; i++, ost_idx = (ost_idx + 1) % ost_count) {
+                rc = lov_check_and_create_object(lov, ost_idx, lsm, req,
+                                                 set->set_oti);
+
+                if (rc == 0)
+                        break;
+        }
+
         RETURN(rc);
 }
 
@@ -534,8 +577,8 @@ static int min_stripe_count(int stripe_cnt, int flags)
                 stripe_cnt - (stripe_cnt / 4) : stripe_cnt);
 }
 
-#define LOV_CREATE_RESEED_MULT 4
-#define LOV_CREATE_RESEED_MIN  1000
+#define LOV_CREATE_RESEED_MULT 30
+#define LOV_CREATE_RESEED_MIN  2000
 /* Allocate objects on osts with round-robin algorithm */
 static int alloc_rr(struct lov_obd *lov, int *idx_arr, int *stripe_cnt,
                     char *poolname, int flags)
@@ -566,7 +609,7 @@ static int alloc_rr(struct lov_obd *lov, int *idx_arr, int *stripe_cnt,
                 GOTO(out, rc);
 
         if (--lqr->lqr_start_count <= 0) {
-                lqr->lqr_start_idx = ll_rand() % osts->op_count;
+                lqr->lqr_start_idx = cfs_rand() % osts->op_count;
                 lqr->lqr_start_count =
                         (LOV_CREATE_RESEED_MIN / max(osts->op_count, 1U) +
                          LOV_CREATE_RESEED_MULT) * max(osts->op_count, 1U);
@@ -837,23 +880,23 @@ static int alloc_qos(struct obd_export *exp, int *idx_arr, int *stripe_cnt,
 
                 if (total_weight) {
 #if BITS_PER_LONG == 32
-                        rand = ll_rand() % (unsigned)total_weight;
+                        rand = cfs_rand() % (unsigned)total_weight;
                         /* If total_weight > 32-bit, first generate the high
                          * 32 bits of the random number, then add in the low
                          * 32 bits (truncated to the upper limit, if needed) */
                         if (total_weight > 0xffffffffULL)
-                                rand = (__u64)(ll_rand() %
+                                rand = (__u64)(cfs_rand() %
                                           (unsigned)(total_weight >> 32)) << 32;
                         else
                                 rand = 0;
 
                         if (rand == (total_weight & 0xffffffff00000000ULL))
-                                rand |= ll_rand() % (unsigned)total_weight;
+                                rand |= cfs_rand() % (unsigned)total_weight;
                         else
-                                rand |= ll_rand();
+                                rand |= cfs_rand();
 
 #else
-                        rand = ((__u64)ll_rand() << 32 | ll_rand()) %
+                        rand = ((__u64)cfs_rand() << 32 | cfs_rand()) %
                                 total_weight;
 #endif
                 } else {
@@ -1005,7 +1048,7 @@ int qos_prep_create(struct obd_export *exp, struct lov_request_set *set)
 
         lsm = set->set_oi->oi_md;
         lsm->lsm_object_id = src_oa->o_id;
-        lsm->lsm_object_gr = src_oa->o_gr;
+        lsm->lsm_object_seq = src_oa->o_seq;
 
         if (!lsm->lsm_stripe_size)
                 lsm->lsm_stripe_size = lov->desc.ld_default_stripe_size;
@@ -1031,7 +1074,7 @@ int qos_prep_create(struct obd_export *exp, struct lov_request_set *set)
                 lov_set_add_req(req, set);
 
                 req->rq_buflen = sizeof(*req->rq_oi.oi_md);
-                OBD_ALLOC(req->rq_oi.oi_md, req->rq_buflen);
+                OBD_ALLOC_LARGE(req->rq_oi.oi_md, req->rq_buflen);
                 if (req->rq_oi.oi_md == NULL)
                         GOTO(out_err, rc = -ENOMEM);