Whamcloud - gitweb
LU-16339 quota: notify OSTs until lge_qunit_nu is set
[fs/lustre-release.git] / lustre / quota / qmt_handler.c
index 3b7b6cf..8eb2824 100644 (file)
@@ -113,7 +113,7 @@ static void qmt_set_id_notify(const struct lu_env *env, struct qmt_device *qmt,
        lqe_gl = lqe->lqe_is_global ? lqe : NULL;
        rc = qmt_pool_lqes_lookup_spec(env, qmt, lqe_rtype(lqe),
                                       lqe_qtype(lqe), &lqe->lqe_id);
-       if (!qti_lqes_cnt(env))
+       if (rc)
                GOTO(lqes_fini, rc);
 
        if (!lqe_gl && qti_lqes_glbl(env)->lqe_is_global)
@@ -122,8 +122,11 @@ static void qmt_set_id_notify(const struct lu_env *env, struct qmt_device *qmt,
        if (!lqe_gl)
                GOTO(lqes_fini, rc);
 
+       mutex_lock(&lqe_gl->lqe_glbl_data_lock);
        if (lqe_gl->lqe_glbl_data)
                qmt_seed_glbe(env, lqe_gl->lqe_glbl_data);
+       mutex_unlock(&lqe_gl->lqe_glbl_data_lock);
+
        /* Even if slaves haven't enqueued quota lock yet,
         * it is needed to set lqe_revoke_time in qmt_id_lock_glimpse
         * in case of reaching qpi_least_qunit */
@@ -156,7 +159,7 @@ int qmt_set_with_lqe(const struct lu_env *env, struct qmt_device *qmt,
        __u64 ver;
        bool dirtied = false;
        int rc = 0;
-       int need_id_notify = 0;
+       bool need_id_notify = false;
        ENTRY;
 
        /* need to write back to global quota file? */
@@ -202,11 +205,15 @@ int qmt_set_with_lqe(const struct lu_env *env, struct qmt_device *qmt,
                /* change quota limits */
                lqe->lqe_hardlimit = hard;
                lqe->lqe_softlimit = soft;
+               if (is_default) {
+                       dirtied = true;
+                       GOTO(quota_write, 0);
+               }
 
 quota_set:
                /* recompute qunit in case it was never initialized */
                if (qmt_revalidate(env, lqe))
-                       need_id_notify = 1;
+                       need_id_notify = true;
 
                /* clear grace time */
                if (lqe->lqe_softlimit == 0 ||
@@ -220,10 +227,18 @@ quota_set:
 
                /* change enforced status based on new parameters */
                if (lqe->lqe_id.qid_uid == 0 || (lqe->lqe_hardlimit == 0 &&
-                   lqe->lqe_softlimit == 0))
-                       lqe->lqe_enforced = false;
-               else
+                   lqe->lqe_softlimit == 0)) {
+                       if (lqe->lqe_enforced) {
+                               lqe->lqe_enforced = false;
+                               /* Clear qunit and edquot as lqe_adjust_edquot
+                                * does not handle not enforced lqes */
+                               lqe->lqe_edquot = 0;
+                               lqe->lqe_qunit = 0;
+                               need_id_notify = true;
+                       }
+               } else {
                        lqe->lqe_enforced = true;
+               }
 
                dirtied = true;
        }
@@ -237,6 +252,7 @@ quota_set:
                dirtied = true;
        }
 
+quota_write:
        if (dirtied) {
                if (!is_updated) {
                        /* write new quota settings to disk */
@@ -252,17 +268,19 @@ quota_set:
 
                /* compute new qunit value now that we have modified the quota
                 * settings or clear/set edquot flag if needed */
-               if (qmt_adjust_qunit(env, lqe) || qmt_adjust_edquot(lqe, now))
-                       need_id_notify |= 1;
+               need_id_notify |= qmt_adjust_qunit(env, lqe);
+               need_id_notify |= qmt_adjust_edquot(lqe, now);
        }
        EXIT;
 out:
        lqe_write_unlock(lqe);
 
 out_nolock:
-       qti_lqes_restore_fini(env);
-       if (th != NULL && !IS_ERR(th))
-               dt_trans_stop(env, qmt->qmt_child, th);
+       if (!is_updated) {
+               if (th != NULL && !IS_ERR(th))
+                       dt_trans_stop(env, qmt->qmt_child, th);
+               qti_lqes_restore_fini(env);
+       }
 
        if (rc == 0 && dirtied) {
                qmt_glb_lock_notify(env, lqe, ver);
@@ -272,14 +290,19 @@ out_nolock:
                        LQUOTA_DEBUG(lqe, "notify all lqe with default quota");
                        iter_data.qeid_env = env;
                        iter_data.qeid_qmt = qmt;
-                       cfs_hash_for_each_safe(lqe->lqe_site->lqs_hash,
+                       cfs_hash_for_each(lqe->lqe_site->lqs_hash,
                                               qmt_entry_iter_cb, &iter_data);
                        /* Always notify slaves with default values. Don't
                         * care about overhead as will be sent only not changed
                         * values(see qmt_id_lock_cb for details).*/
-                       need_id_notify = 1;
+                       need_id_notify = true;
                }
-               if (need_id_notify && !is_updated)
+               /* qti_lqes_inited > 0 means we came here from another
+                * qmt_pool_lqes_lookup(qmt_dqacq, intent_policy ...). Thus
+                * we can't init and add new lqes to don't overwrite already
+                * added.
+                */
+               if (!qti_lqes_inited(env) && need_id_notify)
                        qmt_set_id_notify(env, qmt, lqe);
        }
 
@@ -320,12 +343,205 @@ static int qmt_set(const struct lu_env *env, struct qmt_device *qmt,
        if (IS_ERR(lqe))
                        RETURN(PTR_ERR(lqe));
 
+       lqe->lqe_is_deleted = 0;
+       lqe->lqe_is_reset = 0;
        rc = qmt_set_with_lqe(env, qmt, lqe, hard, soft, time, valid,
                              is_default, is_updated);
+       if (rc == 0)
+               lqe->lqe_is_deleted = 0;
+
+       lqe_putref(lqe);
+       RETURN(rc);
+}
+
+/*
+ * Delete the quota setting of the specified quota ID
+ *
+ * \param env        - is the environment passed by the caller
+ * \param qmt        - is the quota master target
+ * \param restype    - is the pool type, either block (i.e. LQUOTA_RES_DT) or
+ *                     inode (i.e. LQUOTA_RES_MD)
+ * \param qtype      - is the quota type
+ * \param qid        - is the quota indentifier for which we want to delete its
+ *                     quota settings.
+ */
+static int qmt_delete_qid(const struct lu_env *env, struct qmt_device *qmt,
+                         __u8 restype, __u8 qtype, __u64 qid)
+{
+       struct qmt_thread_info *qti = qmt_info(env);
+       union lquota_id *quota_id = &qti->qti_id;
+       struct thandle *th = NULL;
+       struct qmt_pool_info *qpi = NULL;
+       struct lquota_entry *lqe = NULL;
+       __u64 ver = 0;
+       int rc;
+
+       ENTRY;
+
+       quota_id->qid_uid = qid;
+       lqe = qmt_pool_lqe_lookup(env, qmt, restype, qtype, quota_id, NULL);
+       if (IS_ERR(lqe))
+               RETURN(PTR_ERR(lqe));
+
+       lqe_write_lock(lqe);
+
+       qpi = qmt_pool_lookup_glb(env, qmt, restype);
+       if (IS_ERR(qpi))
+               GOTO(out, rc = -ENOMEM);
+
+       th = qmt_trans_start(env, lqe);
+       if (IS_ERR(th))
+               GOTO(out, rc = PTR_ERR(th));
+
+       rc = lquota_disk_delete(env, th,
+                               qpi->qpi_glb_obj[qtype], qid, &ver);
+
+       dt_trans_stop(env, qmt->qmt_child, th);
+
+       if (rc == 0) {
+               lqe_set_deleted(lqe);
+               qmt_glb_lock_notify(env, lqe, ver);
+       } else if (rc == -ENOENT) {
+               rc = 0;
+       }
+
+out:
+       if (!IS_ERR_OR_NULL(qpi))
+               qpi_putref(env, qpi);
+
+       lqe_write_unlock(lqe);
        lqe_putref(lqe);
+
        RETURN(rc);
 }
 
+static int qmt_reset_slv_cb(const struct lu_env *env, struct lu_fid *glb_fid,
+                           char *slv_name, struct lu_fid *slv_fid, void *arg)
+{
+       struct qmt_device *qmt = (struct qmt_device *)arg;
+       struct qmt_thread_info *qti = qmt_info(env);
+       struct dt_object *slv_obj = NULL;
+       struct lquota_slv_rec rec;
+       struct thandle *th = NULL;
+       int rc;
+
+       slv_obj = dt_locate(env, qmt->qmt_child, slv_fid);
+       if (IS_ERR(slv_obj))
+               GOTO(out, rc = PTR_ERR(slv_obj));
+
+       if (slv_obj->do_index_ops == NULL) {
+               rc = slv_obj->do_ops->do_index_try(env, slv_obj,
+                                                  &dt_quota_slv_features);
+               if (rc) {
+                       CERROR("%s: fail to setup slave idx for %s: rc = %d\n",
+                              qmt->qmt_child->dd_lu_dev.ld_obd->obd_name,
+                              slv_name, rc);
+                       GOTO(out, rc);
+               }
+       }
+
+       th = qmt_trans_start(env, qti_lqes(env)[0]);
+       if (IS_ERR(th))
+               GOTO(out, rc = PTR_ERR(th));
+
+       rec.qsr_granted = 0;
+       rc = lquota_disk_write(env, th, slv_obj, &qti->qti_id,
+                              (struct dt_rec *)&rec, 0, NULL);
+       if (rc)
+               CERROR("%s: failed to reset slave grant for %s: rc = %d\n",
+                      qmt->qmt_child->dd_lu_dev.ld_obd->obd_name, slv_name,
+                      rc);
+out:
+       if (!IS_ERR_OR_NULL(th))
+               dt_trans_stop(env, qmt->qmt_child, th);
+
+       if (slv_obj != NULL)
+               dt_object_put(env, slv_obj);
+       return 0;
+}
+
+/*
+ * Reset the quota of the quota ID, it will reset the soft/hard limit and grant
+ *
+ * \param env        - is the environment passed by the caller
+ * \param qmt        - is the quota master target
+ * \param restype    - is the pool type, either block (i.e. LQUOTA_RES_DT) or
+ *                     inode (i.e. LQUOTA_RES_MD)
+ * \param qtype      - is the quota type
+ * \param qid        - is the quota indentifier for which we want to delete its
+ *                     quota settings.
+ */
+static int qmt_reset_qid(const struct lu_env *env, struct qmt_device *qmt,
+                        __u8 restype, __u8 qtype, __u64 qid)
+{
+       struct qmt_thread_info *qti = qmt_info(env);
+       union lquota_id *quota_id = &qti->qti_id;
+       struct qmt_pool_info *qpi = NULL;
+       struct lquota_entry *lqe = NULL;
+       struct thandle *th = NULL;
+       __u64 softlimit = 0, hardlimit = 0;
+       __u64 ver = 0;
+       int rc;
+
+       ENTRY;
+
+       quota_id->qid_uid = qid;
+       lqe = qmt_pool_lqe_lookup(env, qmt, restype, qtype, quota_id, NULL);
+       if (IS_ERR(lqe))
+               RETURN(PTR_ERR(lqe));
+
+       lqe_write_lock(lqe);
+
+       qpi = qmt_pool_lookup_glb(env, qmt, restype);
+       if (IS_ERR(qpi))
+               GOTO(out, rc = -ENOMEM);
+
+       th = qmt_trans_start(env, lqe);
+       if (IS_ERR(th))
+               GOTO(out, rc = PTR_ERR(th));
+
+       softlimit = lqe->lqe_softlimit;
+       hardlimit = lqe->lqe_hardlimit;
+
+       lqe->lqe_softlimit = 0;
+       lqe->lqe_hardlimit = 0;
+       lqe->lqe_granted = 0;
+       lqe->lqe_edquot = 0;
+       lqe->lqe_qunit = 0;
+       lqe->lqe_is_default = 0;
+       lqe->lqe_is_deleted = 0;
+       lqe->lqe_is_reset = 1;
+       rc = qmt_glb_write(env, th, lqe, LQUOTA_BUMP_VER, &ver);
+       if (rc)
+               LQUOTA_ERROR(lqe, "failed to write quota global rec\n");
+       dt_trans_stop(env, qmt->qmt_child, th);
+       if (rc)
+               GOTO(out, rc);
+
+       lquota_generate_fid(&qti->qti_fid, restype, qtype);
+       qti_lqes(env)[0] = lqe;
+       lquota_disk_for_each_slv(env, qpi->qpi_root, &qti->qti_fid,
+                                qmt_reset_slv_cb, qmt);
+
+       qmt_glb_lock_notify(env, lqe, ver);
+
+out:
+       if (rc) {
+               if (softlimit != 0)
+                       lqe->lqe_softlimit = softlimit;
+               if (hardlimit != 0)
+                       lqe->lqe_hardlimit = hardlimit;
+               lqe->lqe_is_reset = 0;
+       }
+
+       if (!IS_ERR_OR_NULL(qpi))
+               qpi_putref(env, qpi);
+
+       lqe_write_unlock(lqe);
+       lqe_putref(lqe);
+
+       RETURN(rc);
+}
 /*
  * Handle quotactl request.
  *
@@ -403,8 +619,9 @@ static int qmt_quotactl(const struct lu_env *env, struct lu_device *ld,
                break;
 
        case LUSTRE_Q_GETDEFAULT:
+       case LUSTRE_Q_GETDEFAULT_POOL:
                is_default = true;
-               /* fallthrough */
+               fallthrough;
 
        case Q_GETQUOTA: /* consult quota limit */
        case LUSTRE_Q_GETQUOTAPOOL:
@@ -437,8 +654,9 @@ static int qmt_quotactl(const struct lu_env *env, struct lu_device *ld,
                break;
 
        case LUSTRE_Q_SETDEFAULT:
+       case LUSTRE_Q_SETDEFAULT_POOL:
                is_default = true;
-               /* fallthrough */
+               fallthrough;
 
        case Q_SETQUOTA: /* change quota limits */
        case LUSTRE_Q_SETQUOTAPOOL:
@@ -465,6 +683,51 @@ static int qmt_quotactl(const struct lu_env *env, struct lu_device *ld,
                                     false, poolname);
                break;
 
+       case LUSTRE_Q_DELETEQID:
+               rc = qmt_delete_qid(env, qmt, LQUOTA_RES_MD, oqctl->qc_type,
+                                   oqctl->qc_id);
+               if (rc)
+                       break;
+
+               rc = qmt_delete_qid(env, qmt, LQUOTA_RES_DT, oqctl->qc_type,
+                                   oqctl->qc_id);
+               break;
+
+       case LUSTRE_Q_RESETQID:
+               if (oqctl->qc_id == 0)
+                       RETURN(-EINVAL);
+
+               id->qid_uid = oqctl->qc_id;
+               /* save the quota setting before resetting */
+               rc = qmt_get(env, qmt, LQUOTA_RES_MD, oqctl->qc_type, id,
+                            &dqb->dqb_ihardlimit, &dqb->dqb_isoftlimit,
+                            &dqb->dqb_itime, false, NULL);
+               if (rc)
+                       break;
+               else
+                       dqb->dqb_valid |= QIF_ILIMITS | QIF_ITIME;
+
+               rc = qmt_get(env, qmt, LQUOTA_RES_DT, oqctl->qc_type, id,
+                            &dqb->dqb_bhardlimit, &dqb->dqb_bsoftlimit,
+                            &dqb->dqb_btime, false, NULL);
+               if (rc)
+                       break;
+
+               dqb->dqb_valid |= QIF_BLIMITS | QIF_BTIME;
+               dqb->dqb_curinodes = 0;
+               dqb->dqb_curspace = 0;
+
+               /* reset the corresponding quota ID */
+               rc = qmt_reset_qid(env, qmt, LQUOTA_RES_MD, oqctl->qc_type,
+                                  oqctl->qc_id);
+               if (rc)
+                       break;
+
+               rc = qmt_reset_qid(env, qmt, LQUOTA_RES_DT, oqctl->qc_type,
+                                  oqctl->qc_id);
+               break;
+
+
        default:
                CERROR("%s: unsupported quotactl command: %d\n",
                       qmt->qmt_svname, oqctl->qc_cmd);
@@ -501,14 +764,22 @@ static inline bool qmt_lqes_can_rel(const struct lu_env *env, __u64 cnt)
        return can_release;
 }
 
-static inline void qmt_rel_lqes(const struct lu_env *env, __u64 *slv, __u64 cnt)
+static inline void qmt_rel_lqes(const struct lu_env *env, __u64 *slv, __u64 cnt,
+                               bool reset)
 {
        int i;
 
-       for (i = 0; i < qti_lqes_cnt(env); i++)
-               qti_lqe_granted(env, i) -= cnt;
+       for (i = 0; i < qti_lqes_cnt(env); i++) {
+               if (reset)
+                       qti_lqe_granted(env, i) = 0;
+               else
+                       qti_lqe_granted(env, i) -= cnt;
+       }
 
-       *slv -= cnt;
+       if (reset)
+               *slv = 0;
+       else
+               *slv -= cnt;
 }
 
 static inline bool qmt_lqes_cannot_grant(const struct lu_env *env, __u64 cnt)
@@ -608,6 +879,7 @@ static inline void qmt_lqes_tune_grace(const struct lu_env *env, __u64 now)
  *                   acquire/release
  * \param qb_usage - is the current space usage on the slave
  * \param repbody - is the quota_body of reply
+ * \param idx     - is the index of a slave target
  *
  * \retval 0            : success
  * \retval -EDQUOT      : out of quota
@@ -616,9 +888,9 @@ static inline void qmt_lqes_tune_grace(const struct lu_env *env, __u64 now)
  */
 int qmt_dqacq0(const struct lu_env *env, struct qmt_device *qmt,
               struct obd_uuid *uuid, __u32 qb_flags, __u64 qb_count,
-              __u64 qb_usage, struct quota_body *repbody)
+              __u64 qb_usage, struct quota_body *repbody, int idx)
 {
-       __u64                    now, count;
+       __u64                    now, count = 0;
        struct dt_object        *slv_obj = NULL;
        __u64                    slv_granted, slv_granted_bck;
        struct thandle          *th = NULL;
@@ -635,6 +907,10 @@ int qmt_dqacq0(const struct lu_env *env, struct qmt_device *qmt,
        if (OBD_FAIL_CHECK(OBD_FAIL_QUOTA_RECOVERABLE_ERR))
                RETURN(-cfs_fail_val);
 
+       if (OBD_FAIL_CHECK(OBD_FAIL_QUOTA_PREACQ) &&
+          (req_is_preacq(qb_flags) || req_is_rel(qb_flags)))
+               RETURN(-EAGAIN);
+
        if (qti_lqes_restore_init(env))
                RETURN(-ENOMEM);
 
@@ -674,6 +950,13 @@ int qmt_dqacq0(const struct lu_env *env, struct qmt_device *qmt,
        if (req_is_acq(qb_flags) && qb_count == 0)
                GOTO(out_locked, rc = 0);
 
+       if (lqe->lqe_is_reset) {
+               lqe->lqe_granted = 0;
+               repbody->qb_count = qb_count;
+               qmt_rel_lqes(env, &slv_granted, qb_count, lqe->lqe_is_reset);
+               GOTO(out_locked, rc = 0);
+       }
+
        /* fetch how much quota space is already granted to this slave */
        rc = qmt_slv_read(env, &lqe->lqe_id, slv_obj, &slv_granted);
        if (rc) {
@@ -702,7 +985,7 @@ int qmt_dqacq0(const struct lu_env *env, struct qmt_device *qmt,
 
                repbody->qb_count = qb_count;
                /* put released space back to global pool */
-               qmt_rel_lqes(env, &slv_granted, qb_count);
+               qmt_rel_lqes(env, &slv_granted, qb_count, lqe->lqe_is_reset);
                GOTO(out_write, rc = 0);
        }
 
@@ -777,6 +1060,9 @@ out_write:
        /* start/stop grace timer if required */
        qmt_lqes_tune_grace(env, now);
 
+       if (OBD_FAIL_CHECK(OBD_FAIL_QUOTA_GRANT))
+               slv_granted = 0xFFFFFFFFFFF00000;
+
        /* Update slave index first since it is easier to roll back */
        ret = qmt_slv_write(env, th, lqe, slv_obj, LQUOTA_BUMP_VER,
                            &repbody->qb_slv_ver, slv_granted);
@@ -813,7 +1099,7 @@ out_write:
         * size according to the total granted & limits. */
 
        /* clear/set edquot flag and notify slaves via glimpse if needed */
-       qmt_adjust_and_notify(env, qmt, now, qb_flags);
+       qmt_adjust_notify_nu(env, qmt, now, qb_flags, idx);
 out_locked:
        LQUOTA_DEBUG_LQES(env, "dqacq ends count:%llu ver:%llu rc:%d",
                     repbody->qb_count, repbody->qb_slv_ver, rc);
@@ -982,7 +1268,8 @@ static int qmt_dqacq(const struct lu_env *env, struct lu_device *ld,
                RETURN(rc);
 
        rc = qmt_dqacq0(env, qmt, uuid, qbody->qb_flags,
-                       qbody->qb_count, qbody->qb_usage, repbody);
+                       qbody->qb_count, qbody->qb_usage, repbody,
+                       qmt_dom(rtype, stype) ? -1 : idx);
 
        if (lustre_handle_is_used(&qbody->qb_lockh))
                /* return current qunit value only to slaves owning an per-ID