- integrate shrinking grants functionality into ofd

diff --git a/lustre/ofd/ofd_grant.c b/lustre/ofd/ofd_grant.c
index 02ee75c..cd22eda 100644 (file)
--- a/lustre/ofd/ofd_grant.c
+++ b/lustre/ofd/ofd_grant.c
@@ -152,7 +152,8 @@ void filter_grant_discard(struct obd_export *exp)
  * We will later calculate the clients new grant and return it. 
  * Caller must hold osfs lock. 
  */
-void filter_grant_incoming(struct obd_export *exp, struct obdo *oa)
+void filter_grant_incoming(const struct lu_env *env, struct obd_export *exp,
+                           struct obdo *oa)
 {
         struct filter_export_data *fed;
         struct filter_device *ofd = filter_exp(exp);
@@ -161,11 +162,8 @@ void filter_grant_incoming(struct obd_export *exp, struct obdo *oa)
 
         LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock);
 
-        /* Check if OST supports grants (OBD_MD_FLGRANT) or allocated block
-         * flag is set.
-         */ 
         if ((oa->o_valid & (OBD_MD_FLBLOCKS|OBD_MD_FLGRANT)) !=
-            (OBD_MD_FLBLOCKS|OBD_MD_FLGRANT)) {
+                                        (OBD_MD_FLBLOCKS|OBD_MD_FLGRANT)) {
                 oa->o_valid &= ~OBD_MD_FLGRANT;
                 EXIT;
                 return;
@@ -175,8 +173,7 @@ void filter_grant_incoming(struct obd_export *exp, struct obdo *oa)
 
         /* Add some margin, since there is a small race if other RPCs arrive
          * out-or-order and have already consumed some grant.  We want to
-         * leave this here in case there is a large error in accounting. 
-         */
+         * leave this here in case there is a large error in accounting. */
         CDEBUG(D_CACHE,
                "%s: cli %s/%p reports grant: "LPU64" dropped: %u, local: %lu\n",
                obd->obd_name, exp->exp_client_uuid.uuid, exp, oa->o_grant,
@@ -185,8 +182,7 @@ void filter_grant_incoming(struct obd_export *exp, struct obdo *oa)
         /* Update our accounting now so that statfs takes it into account.
          * Note that fed_dirty is only approximate and can become incorrect
          * if RPCs arrive out-of-order.  No important calculations depend
-         * on fed_dirty however, but we must check sanity to not assert. 
-         */
+         * on fed_dirty however, but we must check sanity to not assert. */
         if ((long long)oa->o_dirty < 0)
                 oa->o_dirty = 0;
         else if (oa->o_dirty > fed->fed_grant + 4 * FILTER_GRANT_CHUNK)
@@ -207,6 +203,25 @@ void filter_grant_incoming(struct obd_export *exp, struct obdo *oa)
         ofd->ofd_tot_granted -= oa->o_dropped;
         fed->fed_grant -= oa->o_dropped;
         fed->fed_dirty = oa->o_dirty;
+
+        if (oa->o_flags & OBD_FL_SHRINK_GRANT) {
+                obd_size left_space = filter_grant_space_left(env, exp);
+
+                /*Only if left_space < fo_tot_clients * 32M, 
+                 *then the grant space could be shrinked */
+                if (left_space < ofd->ofd_tot_granted_clients * 
+                                 FILTER_GRANT_SHRINK_LIMIT) { 
+                        fed->fed_grant -= oa->o_grant;
+                        ofd->ofd_tot_granted -= oa->o_grant;
+                        CDEBUG(D_CACHE, "%s: cli %s/%p shrink "LPU64
+                               "fed_grant %ld total "LPU64"\n",
+                               obd->obd_name, exp->exp_client_uuid.uuid,
+                               exp, oa->o_grant, fed->fed_grant,
+                               ofd->ofd_tot_granted);
+                        oa->o_grant = 0;
+                }
+        }
+
         if (fed->fed_dirty < 0 || fed->fed_grant < 0 || fed->fed_pending < 0) {
                 CERROR("%s: cli %s/%p dirty %ld pend %ld grant %ld\n",
                        obd->obd_name, exp->exp_client_uuid.uuid, exp,

diff --git a/lustre/ofd/ofd_internal.h b/lustre/ofd/ofd_internal.h
index 704e89e..6a45870 100644 (file)
--- a/lustre/ofd/ofd_internal.h
+++ b/lustre/ofd/ofd_internal.h
@@ -31,6 +31,7 @@
                               OBD_INCOMPAT_COMMON_LR)
 
 #define FILTER_GRANT_CHUNK (2ULL * PTLRPC_MAX_BRW_SIZE)
+#define FILTER_GRANT_SHRINK_LIMIT (16ULL * FILTER_GRANT_CHUNK)
 #define GRANT_FOR_LLOG      16
 
 #define FILTER_RECOVERY_TIMEOUT (obd_timeout * 5 * HZ / 2) /* *waves hands* */
@@ -159,6 +160,7 @@ struct filter_device {
         obd_size                 ofd_tot_dirty;
         obd_size                 ofd_tot_granted;
         obd_size                 ofd_tot_pending;
+        int                      ofd_tot_granted_clients;
 
         /* filter mod data: filter_device wide values */
         int                      ofd_fmd_max_num; /* per ofd filter_mod_data */
@@ -481,7 +483,8 @@ int filter_attr_get(const struct lu_env *env, struct filter_object *fo,
 /* filter_grants.c */
 void filter_grant_discard(struct obd_export *exp);
 void filter_grant_sanity_check(struct obd_device *obd, const char *func);
-void filter_grant_incoming(struct obd_export *exp, struct obdo *oa);
+void filter_grant_incoming(const struct lu_env *env, struct obd_export *exp,
+                           struct obdo *oa);
 obd_size filter_grant_space_left(const struct lu_env *env,
                                  struct obd_export *exp);
 int filter_grant_client_calc(struct obd_export *exp, obd_size *left,

diff --git a/lustre/ofd/ofd_obd.c b/lustre/ofd/ofd_obd.c
index 8c85fd7..ec03ddb 100644 (file)
--- a/lustre/ofd/ofd_obd.c
+++ b/lustre/ofd/ofd_obd.c
@@ -107,6 +107,8 @@ static int filter_parse_connect_data(const struct lu_env *env,
                        LPU64" left: "LPU64"\n", exp->exp_obd->obd_name,
                        exp->exp_client_uuid.uuid, exp,
                        data->ocd_grant, want, left);
+
+                ofd->ofd_tot_granted_clients ++;
         }
 
         if (data->ocd_connect_flags & OBD_CONNECT_INDEX) {
@@ -286,6 +288,22 @@ static int filter_obd_disconnect(struct obd_export *exp)
         RETURN(rc);
 }
 
+/* reverse import is changed, sync all cancels */
+static void filter_revimp_update(struct obd_export *exp)
+{
+        ENTRY;
+
+        LASSERT(exp);
+        class_export_get(exp);
+
+        /* flush any remaining cancel messages out to the target */
+#if 0
+        filter_sync_llogs(exp->exp_obd, exp);
+#endif
+        class_export_put(exp);
+        EXIT;
+}
+
 static int filter_init_export(struct obd_export *exp)
 {
         spin_lock_init(&exp->exp_filter_data.fed_lock);
@@ -337,6 +355,11 @@ static int filter_destroy_export(struct obd_export *exp)
          * UOSS functionality is implemented. */
          filter_fmd_cleanup(exp);
 
+        if (exp->exp_connect_flags & OBD_CONNECT_GRANT_SHRINK) {
+                if (ofd->ofd_tot_granted_clients > 0)
+                        ofd->ofd_tot_granted_clients --;
+        }
+
         if (!(exp->exp_flags & OBD_OPT_FORCE))
                 filter_grant_sanity_check(exp->exp_obd, __FUNCTION__);
 
@@ -366,13 +389,39 @@ static inline int filter_setup_llog_group(struct obd_export *exp,
         return rc;
 }
 
+static int filter_adapt_sptlrpc_conf(struct obd_device *obd, int initial)
+{
+        struct filter_obd       *filter = &obd->u.filter;
+        struct sptlrpc_rule_set  tmp_rset;
+        int                      rc;
+
+        sptlrpc_rule_set_init(&tmp_rset);
+        rc = sptlrpc_conf_target_get_rules(obd, &tmp_rset, initial);
+        if (rc) {
+                CERROR("obd %s: failed get sptlrpc rules: %d\n",
+                       obd->obd_name, rc);
+                return rc;
+        }
+
+        sptlrpc_target_update_exp_flavor(obd, &tmp_rset);
+
+        write_lock(&filter->fo_sptlrpc_lock);
+        sptlrpc_rule_set_free(&filter->fo_sptlrpc_rset);
+        filter->fo_sptlrpc_rset = tmp_rset;
+        write_unlock(&filter->fo_sptlrpc_lock);
+
+        return 0;
+}
+
 static int filter_set_info_async(struct obd_export *exp, __u32 keylen,
                                  void *key, __u32 vallen, void *val,
                                  struct ptlrpc_request_set *set)
 {
         struct filter_device *ofd = filter_exp(exp);
-        struct obd_device *obd;
-        int rc = 0, group;
+        struct obd_device    *obd;
+        int                   rc = 0;
+        int                   group;
+        struct                lu_env env;
         ENTRY;
 
         obd = exp->exp_obd;
@@ -381,15 +430,39 @@ static int filter_set_info_async(struct obd_export *exp, __u32 keylen,
                 RETURN(-EINVAL);
         }
 
+        rc = lu_env_init(&env, LCT_DT_THREAD);
+        if (rc)
+                RETURN(rc);
+
         if (KEY_IS(KEY_CAPA_KEY)) {
                 rc = filter_update_capa_key(ofd, (struct lustre_capa_key *)val);
                 if (rc)
                         CERROR("filter update capability key failed: %d\n", rc);
-                RETURN(rc);
+                GOTO(out, rc);
+        }
+
+        if (KEY_IS(KEY_REVIMP_UPD)) {
+                filter_revimp_update(exp);
+                lquota_clearinfo(filter_quota_interface_ref, exp, exp->exp_obd);
+                GOTO(out, rc = 0);
+        }
+
+        if (KEY_IS(KEY_SPTLRPC_CONF)) {
+                filter_adapt_sptlrpc_conf(obd, 0);
+                GOTO(out, rc = 0);
+        }
+
+        if (KEY_IS(KEY_GRANT_SHRINK)) {
+                struct ost_body *body = (struct ost_body *)val;
+                /* handle shrink grant */
+                spin_lock(&exp->exp_obd->obd_osfs_lock);
+                filter_grant_incoming(&env, exp, &body->oa);
+                spin_unlock(&exp->exp_obd->obd_osfs_lock);
+                GOTO(out, rc = 0);
         }
 
         if (!KEY_IS(KEY_MDS_CONN))
-                RETURN(-EINVAL);
+                GOTO(out, rc = -EINVAL);
 
         LCONSOLE_WARN("%s: received MDS connection from %s\n", obd->obd_name,
                       obd_export_nid2str(exp));
@@ -420,8 +493,9 @@ static int filter_set_info_async(struct obd_export *exp, __u32 keylen,
                 /* setup llog group 1 for interop */
                 filter_setup_llog_group(exp, obd, FILTER_GROUP_LLOG);
         }
-out:
 #endif
+out:
+        lu_env_fini(&env);
         RETURN(rc);
 }