Whamcloud - gitweb
LU-2871 lod: stripe data across the OSTs correctly
[fs/lustre-release.git] / lustre / lod / lod_qos.c
index 5e91e88..a009f8b 100644 (file)
@@ -59,7 +59,7 @@
 #define TGT_BAVAIL(i) (OST_TGT(lod,i)->ltd_statfs.os_bavail * \
                       OST_TGT(lod,i)->ltd_statfs.os_bsize)
 
-int qos_add_tgt(struct lod_device *lod, struct lod_ost_desc *ost_desc)
+int qos_add_tgt(struct lod_device *lod, struct lod_tgt_desc *ost_desc)
 {
        struct lov_qos_oss *oss = NULL, *temposs;
        struct obd_export  *exp = ost_desc->ltd_exp;
@@ -118,7 +118,7 @@ out:
        RETURN(rc);
 }
 
-int qos_del_tgt(struct lod_device *lod, struct lod_ost_desc *ost_desc)
+int qos_del_tgt(struct lod_device *lod, struct lod_tgt_desc *ost_desc)
 {
        struct lov_qos_oss *oss;
        int                 rc = 0;
@@ -148,7 +148,7 @@ out:
 static int lod_statfs_and_check(const struct lu_env *env, struct lod_device *d,
                                int index, struct obd_statfs *sfs)
 {
-       struct lod_ost_desc *ost;
+       struct lod_tgt_desc *ost;
        int                  rc;
 
        LASSERT(d);
@@ -175,7 +175,9 @@ static int lod_statfs_and_check(const struct lu_env *env, struct lod_device *d,
                spin_unlock(&d->lod_desc_lock);
        } else if (rc == 0 && ost->ltd_active == 0) {
                /* turned active? */
-               LASSERT(d->lod_desc.ld_active_tgt_count < d->lod_ostnr);
+               LASSERTF(d->lod_desc.ld_active_tgt_count < d->lod_ostnr,
+                        "active tgt count %d, ost nr %d\n",
+                        d->lod_desc.ld_active_tgt_count, d->lod_ostnr);
                spin_lock(&d->lod_desc_lock);
                if (ost->ltd_active == 0) {
                        ost->ltd_active = 1;
@@ -188,7 +190,7 @@ static int lod_statfs_and_check(const struct lu_env *env, struct lod_device *d,
                spin_unlock(&d->lod_desc_lock);
        }
 
-       return rc;
+       RETURN(rc);
 }
 
 static void lod_qos_statfs_update(const struct lu_env *env,
@@ -225,6 +227,7 @@ static void lod_qos_statfs_update(const struct lu_env *env,
 
 out:
        up_write(&lod->lod_qos.lq_rw_sem);
+       EXIT;
 }
 
 /* Recalculate per-object penalties for OSSs and OSTs,
@@ -363,7 +366,7 @@ static int lod_qos_calc_weight(struct lod_device *lod, int i)
 static int lod_qos_used(struct lod_device *lod, struct ost_pool *osts,
                        __u32 index, __u64 *total_wt)
 {
-       struct lod_ost_desc *ost;
+       struct lod_tgt_desc *ost;
        struct lov_qos_oss  *oss;
        int j;
        ENTRY;
@@ -443,7 +446,7 @@ static int lod_qos_calc_rr(struct lod_device *lod, struct ost_pool *src_pool,
                           struct lov_qos_rr *lqr)
 {
        struct lov_qos_oss  *oss;
-       struct lod_ost_desc *ost;
+       struct lod_tgt_desc *ost;
        unsigned placed, real_count;
        int i, rc;
        ENTRY;
@@ -549,7 +552,7 @@ static struct dt_object *lod_qos_declare_object_on(const struct lu_env *env,
                                                   int ost_idx,
                                                   struct thandle *th)
 {
-       struct lod_ost_desc *ost;
+       struct lod_tgt_desc *ost;
        struct lu_object *o, *n;
        struct lu_device *nd;
        struct dt_object *dt;
@@ -629,7 +632,7 @@ static inline int lod_qos_ost_in_use_clear(const struct lu_env *env, int stripes
                CERROR("can't allocate memory for ost-in-use array\n");
                return -ENOMEM;
        }
-       memset(info->lti_ea_store, 0, sizeof(int) * stripes);
+       memset(info->lti_ea_store, -1, sizeof(int) * stripes);
        return 0;
 }
 
@@ -657,7 +660,8 @@ static int lod_qos_is_ost_used(const struct lu_env *env, int ost, int stripes)
 
 /* Allocate objects on OSTs with round-robin algorithm */
 static int lod_alloc_rr(const struct lu_env *env, struct lod_object *lo,
-                       int flags, struct thandle *th)
+                       struct dt_object **stripe, int flags,
+                       struct thandle *th)
 {
        struct lod_device *m = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
        struct obd_statfs *sfs = &lod_env_info(env)->lti_osfs;
@@ -790,7 +794,7 @@ repeat_find:
                 * We've successfuly declared (reserved) an object
                 */
                lod_qos_ost_in_use(env, stripe_idx, ost_idx);
-               lo->ldo_stripe[stripe_idx] = o;
+               stripe[stripe_idx] = o;
                stripe_idx++;
 
        }
@@ -824,7 +828,8 @@ out:
 
 /* alloc objects on osts with specific stripe offset */
 static int lod_alloc_specific(const struct lu_env *env, struct lod_object *lo,
-                             int flags, struct thandle *th)
+                             struct dt_object **stripe, int flags,
+                             struct thandle *th)
 {
        struct lod_device *m = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
        struct obd_statfs *sfs = &lod_env_info(env)->lti_osfs;
@@ -916,7 +921,8 @@ repeat_find:
                /*
                 * We've successfuly declared (reserved) an object
                 */
-               lo->ldo_stripe[stripe_num] = o;
+               lod_qos_ost_in_use(env, stripe_num, ost_idx);
+               stripe[stripe_num] = o;
                stripe_num++;
 
                /* We have enough stripes */
@@ -971,11 +977,12 @@ static inline int lod_qos_is_usable(struct lod_device *lod)
    - network resources (shared OSS's)
  */
 static int lod_alloc_qos(const struct lu_env *env, struct lod_object *lo,
-                        int flags, struct thandle *th)
+                        struct dt_object **stripe, int flags,
+                        struct thandle *th)
 {
        struct lod_device   *m = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
        struct obd_statfs   *sfs = &lod_env_info(env)->lti_osfs;
-       struct lod_ost_desc *ost;
+       struct lod_tgt_desc *ost;
        struct dt_object    *o;
        __u64                total_weight = 0;
        int                  nfound, good_osts, i, rc = 0;
@@ -1132,7 +1139,7 @@ static int lod_alloc_qos(const struct lu_env *env, struct lod_object *lo,
                                          idx, (int) PTR_ERR(o));
                                continue;
                        }
-                       lo->ldo_stripe[nfound++] = o;
+                       stripe[nfound++] = o;
                        lod_qos_used(m, osts, idx, &total_weight);
                        rc = 0;
                        break;
@@ -1154,9 +1161,9 @@ static int lod_alloc_qos(const struct lu_env *env, struct lod_object *lo,
                 */
                LCONSOLE_INFO("wanted %d, found %d\n", stripe_cnt, nfound);
                for (i = 0; i < nfound; i++) {
-                       LASSERT(lo->ldo_stripe[i]);
-                       lu_object_put(env, &lo->ldo_stripe[i]->do_lu);
-                       lo->ldo_stripe[i] = NULL;
+                       LASSERT(stripe[i] != NULL);
+                       lu_object_put(env, &stripe[i]->do_lu);
+                       stripe[i] = NULL;
                }
 
                /* makes sense to rebalance next time */
@@ -1264,9 +1271,11 @@ static int lod_qos_parse_config(const struct lu_env *env,
 
        if (magic == __swab32(LOV_USER_MAGIC_V1)) {
                lustre_swab_lov_user_md_v1(v1);
+               magic = v1->lmm_magic;
        } else if (magic == __swab32(LOV_USER_MAGIC_V3)) {
                v3 = buf->lb_buf;
                lustre_swab_lov_user_md_v3(v3);
+               magic = v3->lmm_magic;
        }
 
        if (unlikely(magic != LOV_MAGIC_V1 && magic != LOV_MAGIC_V3)) {
@@ -1313,6 +1322,9 @@ static int lod_qos_parse_config(const struct lu_env *env,
                v3 = buf->lb_buf;
                lod_object_set_pool(lo, v3->lmm_pool_name);
 
+               /* In the function below, .hs_keycmp resolves to
+                * pool_hashkey_keycmp() */
+               /* coverity[overrun-buffer-val] */
                pool = lod_find_pool(d, v3->lmm_pool_name);
                if (pool != NULL) {
                        if (lo->ldo_def_stripe_offset !=
@@ -1344,9 +1356,11 @@ int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo,
                        struct lu_attr *attr, const struct lu_buf *buf,
                        struct thandle *th)
 {
-       struct lod_device *d = lu2lod_dev(lod2lu_obj(lo)->lo_dev);
-       int flag = LOV_USES_ASSIGNED_STRIPE;
-       int i, rc = 0;
+       struct lod_device      *d = lu2lod_dev(lod2lu_obj(lo)->lo_dev);
+       struct dt_object      **stripe;
+       int                     stripe_len;
+       int                     flag = LOV_USES_ASSIGNED_STRIPE;
+       int                     i, rc;
        ENTRY;
 
        LASSERT(lo);
@@ -1374,24 +1388,42 @@ int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo,
                 * no striping has been created so far
                 */
                LASSERT(lo->ldo_stripenr > 0);
+               /*
+                * statfs and check OST targets now, since ld_active_tgt_count
+                * could be changed if some OSTs are [de]activated manually.
+                */
+               lod_qos_statfs_update(env, d);
                lo->ldo_stripenr = lod_get_stripecnt(d, LOV_MAGIC,
                                lo->ldo_stripenr);
-               i = sizeof(struct dt_object *) * lo->ldo_stripenr;
-               OBD_ALLOC(lo->ldo_stripe, i);
-               if (lo->ldo_stripe == NULL)
+
+               stripe_len = lo->ldo_stripenr;
+               OBD_ALLOC(stripe, sizeof(stripe[0]) * stripe_len);
+               if (stripe == NULL)
                        GOTO(out, rc = -ENOMEM);
-               lo->ldo_stripes_allocated = lo->ldo_stripenr;
 
-               lod_getref(d);
+               lod_getref(&d->lod_ost_descs);
                /* XXX: support for non-0 files w/o objects */
+               CDEBUG(D_OTHER, "tgt_count %d stripenr %d\n",
+                               d->lod_desc.ld_tgt_count, stripe_len);
                if (lo->ldo_def_stripe_offset >= d->lod_desc.ld_tgt_count) {
-                       lod_qos_statfs_update(env, d);
-                       rc = lod_alloc_qos(env, lo, flag, th);
+                       rc = lod_alloc_qos(env, lo, stripe, flag, th);
                        if (rc == -EAGAIN)
-                               rc = lod_alloc_rr(env, lo, flag, th);
-               } else
-                       rc = lod_alloc_specific(env, lo, flag, th);
-               lod_putref(d);
+                               rc = lod_alloc_rr(env, lo, stripe, flag, th);
+               } else {
+                       rc = lod_alloc_specific(env, lo, stripe, flag, th);
+               }
+               lod_putref(d, &d->lod_ost_descs);
+
+               if (rc < 0) {
+                       for (i = 0; i < stripe_len; i++)
+                               if (stripe[i] != NULL)
+                                       lu_object_put(env, &stripe[i]->do_lu);
+
+                       OBD_FREE(stripe, sizeof(stripe[0]) * stripe_len);
+               } else {
+                       lo->ldo_stripe = stripe;
+                       lo->ldo_stripes_allocated = stripe_len;
+               }
        } else {
                /*
                 * lod_qos_parse_config() found supplied buf as a predefined