Holding lprocfs lock with send rpc can produce block for destroy

obd objects and this also block reconnect with -EALREADY.
This isn't fix all lprocfs bugs - but make it rare.
b=14533
i=adilger
i=nathan

diff --git a/lustre/ChangeLog b/lustre/ChangeLog
index 2ca6ce4..b93151e 100644 (file)
--- a/lustre/ChangeLog
+++ b/lustre/ChangeLog
@@ -12,6 +12,14 @@ tbd  Sun Microsystems, Inc.
        * RHEL 4 and RHEL 5/SLES 10 clients behaves differently on 'cd' to a
         removed cwd "./" (refer to Bugzilla 14399).
 
        * RHEL 4 and RHEL 5/SLES 10 clients behaves differently on 'cd' to a
         removed cwd "./" (refer to Bugzilla 14399).
 

+Severity   : normal
+Bugzilla   : 14533
+Frequency  : rare, on recovery
+Description: read procfs can produce deadlock in some situation
+Details    : Holding lprocfs lock which send rpc can produce block for destroy
+             obd objects and this also block reconnect with -EALREADY. This isn't
+             fix all lprocfs bugs - but make it rare.
+

 Severity   : enhancement
 Bugzilla   : 15152
 Description: Update kernel to RHEL5 2.6.18-53.1.14.el5.
 Severity   : enhancement
 Bugzilla   : 15152
 Description: Update kernel to RHEL5 2.6.18-53.1.14.el5.

diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h
index d79772c..65e1f91 100644 (file)
--- a/lustre/include/lustre/lustre_idl.h
+++ b/lustre/include/lustre/lustre_idl.h
@@ -871,6 +871,8 @@ struct obd_statfs {
 };
 
 extern void lustre_swab_obd_statfs (struct obd_statfs *os);
 };
 
 extern void lustre_swab_obd_statfs (struct obd_statfs *os);

+#define OBD_STATFS_NODELAY      0x0001  /* requests should be send without delay
+                                         * and resends for avoid deadlocks */

 
 /* ost_body.data values for OST_BRW */
 
 
 /* ost_body.data values for OST_BRW */
 

diff --git a/lustre/include/obd.h b/lustre/include/obd.h
index ce00d17..953192c 100644 (file)
--- a/lustre/include/obd.h
+++ b/lustre/include/obd.h
@@ -142,8 +142,11 @@ struct obd_info {
          * OSC. (e.g. lov_prep_enqueue_set initialises extent of the policy,
          * and osc_enqueue passes it into ldlm_lock_match & ldlm_cli_enqueue. */
         ldlm_policy_data_t      oi_policy;
          * OSC. (e.g. lov_prep_enqueue_set initialises extent of the policy,
          * and osc_enqueue passes it into ldlm_lock_match & ldlm_cli_enqueue. */
         ldlm_policy_data_t      oi_policy;

-        /* Flags used while lock handling. The flags obtained on the enqueue
-         * request are set here, therefore they are request specific. */
+        /* Flags used for set request specific flags:
+           - while lock handling, the flags obtained on the enqueue
+           request are set here.
+           - while stats, the flags used for control delay/resend.
+         */

         int                     oi_flags;
         /* Lock handle specific for every OSC lock. */
         struct lustre_handle   *oi_lockh;
         int                     oi_flags;
         /* Lock handle specific for every OSC lock. */
         struct lustre_handle   *oi_lockh;


@@ -1091,7 +1094,7 @@ struct obd_ops {
         int (*o_fid_delete)(struct obd_export *exp, const struct lu_fid *fid);
 
         int (*o_statfs)(struct obd_device *obd, struct obd_statfs *osfs,
         int (*o_fid_delete)(struct obd_export *exp, const struct lu_fid *fid);
 
         int (*o_statfs)(struct obd_device *obd, struct obd_statfs *osfs,

-                        __u64 max_age);
+                        __u64 max_age, __u32 flags);

         int (*o_statfs_async)(struct obd_device *obd, struct obd_info *oinfo,
                               __u64 max_age, struct ptlrpc_request_set *set);
         int (*o_packmd)(struct obd_export *exp, struct lov_mds_md **disk_tgt,
         int (*o_statfs_async)(struct obd_device *obd, struct obd_info *oinfo,
                               __u64 max_age, struct ptlrpc_request_set *set);
         int (*o_packmd)(struct obd_export *exp, struct lov_mds_md **disk_tgt,

diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h
index f3f9c4d..30a3f7f 100644 (file)
--- a/lustre/include/obd_class.h
+++ b/lustre/include/obd_class.h
@@ -1008,7 +1008,7 @@ static inline int obd_statfs_async(struct obd_device *obd,
 
 static inline int obd_statfs_rqset(struct obd_device *obd,
                                    struct obd_statfs *osfs, __u64 max_age,
 
 static inline int obd_statfs_rqset(struct obd_device *obd,
                                    struct obd_statfs *osfs, __u64 max_age,

-                                   int quick_pry)
+                                   __u32 flags)

 {
         struct ptlrpc_request_set *set = NULL;
         struct obd_info oinfo = { { { 0 } } };
 {
         struct ptlrpc_request_set *set = NULL;
         struct obd_info oinfo = { { { 0 } } };


@@ -1020,20 +1020,10 @@ static inline int obd_statfs_rqset(struct obd_device *obd,
                 RETURN(-ENOMEM);
 
         oinfo.oi_osfs = osfs;
                 RETURN(-ENOMEM);
 
         oinfo.oi_osfs = osfs;

+        oinfo.oi_flags = flags;

         rc = obd_statfs_async(obd, &oinfo, max_age, set);
         rc = obd_statfs_async(obd, &oinfo, max_age, set);

-        if (rc == 0) {
-                struct ptlrpc_request *req;
-
-                if (quick_pry)
-                        list_for_each_entry(req, &set->set_requests,
-                                            rq_set_chain) {
-                                spin_lock(&req->rq_lock);
-                                req->rq_no_resend = 1;
-                                req->rq_no_delay  = 1;
-                                spin_unlock(&req->rq_lock);
-                        }
+        if (rc == 0)

                 rc = ptlrpc_set_wait(set);
                 rc = ptlrpc_set_wait(set);

-        }

         ptlrpc_set_destroy(set);
         RETURN(rc);
 }
         ptlrpc_set_destroy(set);
         RETURN(rc);
 }


@@ -1042,7 +1032,7 @@ static inline int obd_statfs_rqset(struct obd_device *obd,
  * If the cache is older than @max_age we will get a new value from the
  * target.  Use a value of "cfs_time_current() + HZ" to guarantee freshness. */
 static inline int obd_statfs(struct obd_device *obd, struct obd_statfs *osfs,
  * If the cache is older than @max_age we will get a new value from the
  * target.  Use a value of "cfs_time_current() + HZ" to guarantee freshness. */
 static inline int obd_statfs(struct obd_device *obd, struct obd_statfs *osfs,

-                             __u64 max_age)
+                             __u64 max_age, __u32 flags)

 {
         int rc = 0;
         ENTRY;
 {
         int rc = 0;
         ENTRY;


@@ -1056,7 +1046,7 @@ static inline int obd_statfs(struct obd_device *obd, struct obd_statfs *osfs,
         CDEBUG(D_SUPER, "osfs "LPU64", max_age "LPU64"\n",
                obd->obd_osfs_age, max_age);
         if (cfs_time_before_64(obd->obd_osfs_age, max_age)) {
         CDEBUG(D_SUPER, "osfs "LPU64", max_age "LPU64"\n",
                obd->obd_osfs_age, max_age);
         if (cfs_time_before_64(obd->obd_osfs_age, max_age)) {

-                rc = OBP(obd, statfs)(obd, osfs, max_age);
+                rc = OBP(obd, statfs)(obd, osfs, max_age, flags);

                 if (rc == 0) {
                         spin_lock(&obd->obd_osfs_lock);
                         memcpy(&obd->obd_osfs, osfs, sizeof(obd->obd_osfs));
                 if (rc == 0) {
                         spin_lock(&obd->obd_osfs_lock);
                         memcpy(&obd->obd_osfs, osfs, sizeof(obd->obd_osfs));

diff --git a/lustre/ldlm/ldlm_resource.c b/lustre/ldlm/ldlm_resource.c
index b87ab5f..8d914a0 100644 (file)
--- a/lustre/ldlm/ldlm_resource.c
+++ b/lustre/ldlm/ldlm_resource.c
@@ -478,6 +478,19 @@ int ldlm_namespace_free_prior(struct ldlm_namespace *ns)
         if (!ns)
                 RETURN(ELDLM_OK);
 
         if (!ns)
                 RETURN(ELDLM_OK);
 

+#ifdef LPROCFS
+        {
+                struct proc_dir_entry *dir;
+                dir = lprocfs_srch(ldlm_ns_proc_dir, ns->ns_name);
+                if (dir == NULL) {
+                        CERROR("dlm namespace %s has no procfs dir?\n",
+                               ns->ns_name);
+                } else {
+                        lprocfs_remove(&dir);
+                }
+        }
+#endif
+

         mutex_down(ldlm_namespace_lock(ns->ns_client));
         list_del(&ns->ns_list_chain);
         atomic_dec(ldlm_namespace_nr(ns->ns_client));
         mutex_down(ldlm_namespace_lock(ns->ns_client));
         list_del(&ns->ns_list_chain);
         atomic_dec(ldlm_namespace_nr(ns->ns_client));


@@ -516,18 +529,6 @@ int ldlm_namespace_free_post(struct ldlm_namespace *ns, int force)
         if (!ns)
                 RETURN(ELDLM_OK);
 
         if (!ns)
                 RETURN(ELDLM_OK);
 

-#ifdef LPROCFS
-        {
-                struct proc_dir_entry *dir;
-                dir = lprocfs_srch(ldlm_ns_proc_dir, ns->ns_name);
-                if (dir == NULL) {
-                        CERROR("dlm namespace %s has no procfs dir?\n",
-                               ns->ns_name);
-                } else {
-                        lprocfs_remove(&dir);
-                }
-        }
-#endif

         OBD_VFREE(ns->ns_hash, sizeof(*ns->ns_hash) * RES_HASH_SIZE);
         OBD_FREE(ns->ns_name, strlen(ns->ns_name) + 1);
         OBD_FREE_PTR(ns);
         OBD_VFREE(ns->ns_hash, sizeof(*ns->ns_hash) * RES_HASH_SIZE);
         OBD_FREE(ns->ns_name, strlen(ns->ns_name) + 1);
         OBD_FREE_PTR(ns);

diff --git a/lustre/liblustre/super.c b/lustre/liblustre/super.c
index 890d977..4fdb76d 100644 (file)
--- a/lustre/liblustre/super.c
+++ b/lustre/liblustre/super.c
@@ -1192,7 +1192,7 @@ static int llu_statfs_internal(struct llu_sb_info *sbi,
         int rc;
         ENTRY;
 
         int rc;
         ENTRY;
 

-        rc = obd_statfs(class_exp2obd(sbi->ll_md_exp), osfs, max_age);
+        rc = obd_statfs(class_exp2obd(sbi->ll_md_exp), osfs, max_age, 0);

         if (rc) {
                 CERROR("md_statfs fails: rc = %d\n", rc);
                 RETURN(rc);
         if (rc) {
                 CERROR("md_statfs fails: rc = %d\n", rc);
                 RETURN(rc);


@@ -2087,7 +2087,7 @@ llu_fsswop_mount(const char *source,
         }
         sbi->ll_md_exp = class_conn2export(&md_conn);
 
         }
         sbi->ll_md_exp = class_conn2export(&md_conn);
 

-        err = obd_statfs(obd, &osfs, 100000000);
+        err = obd_statfs(obd, &osfs, 100000000, 0);

         if (err)
                 GOTO(out_md, err);
 
         if (err)
                 GOTO(out_md, err);
 

diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h
index de34d52..1bffef4 100644 (file)
--- a/lustre/llite/llite_internal.h
+++ b/lustre/llite/llite_internal.h
@@ -633,7 +633,7 @@ int ll_statfs(struct super_block *sb, struct kstatfs *sfs);
 int ll_statfs(struct dentry *de, struct kstatfs *sfs);
 #endif
 int ll_statfs_internal(struct super_block *sb, struct obd_statfs *osfs,
 int ll_statfs(struct dentry *de, struct kstatfs *sfs);
 #endif
 int ll_statfs_internal(struct super_block *sb, struct obd_statfs *osfs,

-                       __u64 max_age);
+                       __u64 max_age, __u32 flags);

 void ll_update_inode(struct inode *inode, struct lustre_md *md);
 void ll_read_inode2(struct inode *inode, void *opaque);
 void ll_delete_inode(struct inode *inode);
 void ll_update_inode(struct inode *inode, struct lustre_md *md);
 void ll_read_inode2(struct inode *inode, void *opaque);
 void ll_delete_inode(struct inode *inode);

diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c
index 86ab3b7..43651c6 100644 (file)
--- a/lustre/llite/llite_lib.c
+++ b/lustre/llite/llite_lib.c
@@ -259,7 +259,7 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt)
                 GOTO(out_md, err);
         }
 
                 GOTO(out_md, err);
         }
 

-        err = obd_statfs(obd, &osfs, cfs_time_current_64() - HZ);
+        err = obd_statfs(obd, &osfs, cfs_time_current_64() - HZ, 0);

         if (err)
                 GOTO(out_md_fid, err);
 
         if (err)
                 GOTO(out_md_fid, err);
 


@@ -1447,14 +1447,14 @@ int ll_setattr(struct dentry *de, struct iattr *attr)
 }
 
 int ll_statfs_internal(struct super_block *sb, struct obd_statfs *osfs,
 }
 
 int ll_statfs_internal(struct super_block *sb, struct obd_statfs *osfs,

-                       __u64 max_age)
+                       __u64 max_age, __u32 flags)

 {
         struct ll_sb_info *sbi = ll_s2sbi(sb);
         struct obd_statfs obd_osfs;
         int rc;
         ENTRY;
 
 {
         struct ll_sb_info *sbi = ll_s2sbi(sb);
         struct obd_statfs obd_osfs;
         int rc;
         ENTRY;
 

-        rc = obd_statfs(class_exp2obd(sbi->ll_md_exp), osfs, max_age);
+        rc = obd_statfs(class_exp2obd(sbi->ll_md_exp), osfs, max_age, flags);

         if (rc) {
                 CERROR("md_statfs fails: rc = %d\n", rc);
                 RETURN(rc);
         if (rc) {
                 CERROR("md_statfs fails: rc = %d\n", rc);
                 RETURN(rc);


@@ -1466,7 +1466,7 @@ int ll_statfs_internal(struct super_block *sb, struct obd_statfs *osfs,
                osfs->os_bavail, osfs->os_blocks, osfs->os_ffree,osfs->os_files);
 
         rc = obd_statfs_rqset(class_exp2obd(sbi->ll_dt_exp),
                osfs->os_bavail, osfs->os_blocks, osfs->os_ffree,osfs->os_files);
 
         rc = obd_statfs_rqset(class_exp2obd(sbi->ll_dt_exp),

-                              &obd_osfs, max_age, 0);
+                              &obd_osfs, max_age, flags);

         if (rc) {
                 CERROR("obd_statfs fails: rc = %d\n", rc);
                 RETURN(rc);
         if (rc) {
                 CERROR("obd_statfs fails: rc = %d\n", rc);
                 RETURN(rc);


@@ -1510,7 +1510,7 @@ int ll_statfs(struct dentry *de, struct kstatfs *sfs)
         /* For now we will always get up-to-date statfs values, but in the
          * future we may allow some amount of caching on the client (e.g.
          * from QOS or lprocfs updates). */
         /* For now we will always get up-to-date statfs values, but in the
          * future we may allow some amount of caching on the client (e.g.
          * from QOS or lprocfs updates). */

-        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - 1);
+        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - 1, 0);

         if (rc)
                 return rc;
 
         if (rc)
                 return rc;
 

diff --git a/lustre/llite/lproc_llite.c b/lustre/llite/lproc_llite.c
index 9ed89be..5417c37 100644 (file)
--- a/lustre/llite/lproc_llite.c
+++ b/lustre/llite/lproc_llite.c
@@ -47,7 +47,8 @@ static int ll_rd_blksize(char *page, char **start, off_t off, int count,
         int rc;
 
         LASSERT(sb != NULL);
         int rc;
 
         LASSERT(sb != NULL);

-        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ);
+        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ,
+                                OBD_STATFS_NODELAY);

         if (!rc) {
               *eof = 1;
               rc = snprintf(page, count, "%u\n", osfs.os_bsize);
         if (!rc) {
               *eof = 1;
               rc = snprintf(page, count, "%u\n", osfs.os_bsize);


@@ -64,7 +65,8 @@ static int ll_rd_kbytestotal(char *page, char **start, off_t off, int count,
         int rc;
 
         LASSERT(sb != NULL);
         int rc;
 
         LASSERT(sb != NULL);

-        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ);
+        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ,
+                                OBD_STATFS_NODELAY);

         if (!rc) {
                 __u32 blk_size = osfs.os_bsize >> 10;
                 __u64 result = osfs.os_blocks;
         if (!rc) {
                 __u32 blk_size = osfs.os_bsize >> 10;
                 __u64 result = osfs.os_blocks;


@@ -87,7 +89,8 @@ static int ll_rd_kbytesfree(char *page, char **start, off_t off, int count,
         int rc;
 
         LASSERT(sb != NULL);
         int rc;
 
         LASSERT(sb != NULL);

-        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ);
+        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ,
+                                OBD_STATFS_NODELAY);

         if (!rc) {
                 __u32 blk_size = osfs.os_bsize >> 10;
                 __u64 result = osfs.os_bfree;
         if (!rc) {
                 __u32 blk_size = osfs.os_bsize >> 10;
                 __u64 result = osfs.os_bfree;


@@ -109,7 +112,8 @@ static int ll_rd_kbytesavail(char *page, char **start, off_t off, int count,
         int rc;
 
         LASSERT(sb != NULL);
         int rc;
 
         LASSERT(sb != NULL);

-        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ);
+        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ,
+                                OBD_STATFS_NODELAY);

         if (!rc) {
                 __u32 blk_size = osfs.os_bsize >> 10;
                 __u64 result = osfs.os_bavail;
         if (!rc) {
                 __u32 blk_size = osfs.os_bsize >> 10;
                 __u64 result = osfs.os_bavail;


@@ -131,7 +135,8 @@ static int ll_rd_filestotal(char *page, char **start, off_t off, int count,
         int rc;
 
         LASSERT(sb != NULL);
         int rc;
 
         LASSERT(sb != NULL);

-        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ);
+        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ,
+                                OBD_STATFS_NODELAY);

         if (!rc) {
                  *eof = 1;
                  rc = snprintf(page, count, LPU64"\n", osfs.os_files);
         if (!rc) {
                  *eof = 1;
                  rc = snprintf(page, count, LPU64"\n", osfs.os_files);


@@ -147,7 +152,8 @@ static int ll_rd_filesfree(char *page, char **start, off_t off, int count,
         int rc;
 
         LASSERT(sb != NULL);
         int rc;
 
         LASSERT(sb != NULL);

-        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ);
+        rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ,
+                                OBD_STATFS_NODELAY);

         if (!rc) {
                  *eof = 1;
                  rc = snprintf(page, count, LPU64"\n", osfs.os_ffree);
         if (!rc) {
                  *eof = 1;
                  rc = snprintf(page, count, LPU64"\n", osfs.os_ffree);

diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c
index cc42113..34e539c 100644 (file)
--- a/lustre/lmv/lmv_obd.c
+++ b/lustre/lmv/lmv_obd.c
@@ -714,7 +714,7 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp,
 
                 if ((index >= lmv->desc.ld_tgt_count))
                         RETURN(-ENODEV);
 
                 if ((index >= lmv->desc.ld_tgt_count))
                         RETURN(-ENODEV);

-                
+

                 if (!lmv->tgts[index].ltd_active)
                         RETURN(-ENODATA);
 
                 if (!lmv->tgts[index].ltd_active)
                         RETURN(-ENODATA);
 


@@ -723,7 +723,8 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp,
                         RETURN(-EINVAL);
 
                 /* got statfs data */
                         RETURN(-EINVAL);
 
                 /* got statfs data */

-                rc = obd_statfs(mdc_obd, &stat_buf, cfs_time_current_64() - 1);
+                rc = obd_statfs(mdc_obd, &stat_buf,
+                                cfs_time_current_64() - HZ, 0);

                 if (rc)
                         RETURN(rc);
                 if (copy_to_user(data->ioc_pbuf1, &stat_buf, data->ioc_plen1))
                 if (rc)
                         RETURN(rc);
                 if (copy_to_user(data->ioc_pbuf1, &stat_buf, data->ioc_plen1))


@@ -1075,7 +1076,7 @@ out:
 }
 
 static int lmv_statfs(struct obd_device *obd, struct obd_statfs *osfs,
 }
 
 static int lmv_statfs(struct obd_device *obd, struct obd_statfs *osfs,

-                      __u64 max_age)
+                      __u64 max_age, __u32 flags)

 {
         struct lmv_obd *lmv = &obd->u.lmv;
         struct obd_statfs *temp;
 {
         struct lmv_obd *lmv = &obd->u.lmv;
         struct obd_statfs *temp;


@@ -1094,7 +1095,8 @@ static int lmv_statfs(struct obd_device *obd, struct obd_statfs *osfs,
                 if (lmv->tgts[i].ltd_exp == NULL)
                         continue;
 
                 if (lmv->tgts[i].ltd_exp == NULL)
                         continue;
 

-                rc = obd_statfs(lmv->tgts[i].ltd_exp->exp_obd, temp, max_age);
+                rc = obd_statfs(lmv->tgts[i].ltd_exp->exp_obd, temp,
+                                max_age, flags);

                 if (rc) {
                         CERROR("can't stat MDS #%d (%s), error %d\n", i,
                                lmv->tgts[i].ltd_exp->exp_obd->obd_name,
                 if (rc) {
                         CERROR("can't stat MDS #%d (%s), error %d\n", i,
                                lmv->tgts[i].ltd_exp->exp_obd->obd_name,

diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c
index 844fc68..7b71280 100644 (file)
--- a/lustre/lov/lov_obd.c
+++ b/lustre/lov/lov_obd.c
@@ -1024,7 +1024,7 @@ static int lov_create(struct obd_export *exp, struct obdo *src_oa,
         }
 
         maxage = cfs_time_shift_64(-lov->desc.ld_qos_maxage);
         }
 
         maxage = cfs_time_shift_64(-lov->desc.ld_qos_maxage);

-        obd_statfs_rqset(exp->exp_obd, &osfs, maxage, 1);
+        obd_statfs_rqset(exp->exp_obd, &osfs, maxage, OBD_STATFS_NODELAY);

 
         rc = lov_prep_create_set(exp, &oinfo, ea, src_oa, oti, &set);
         if (rc)
 
         rc = lov_prep_create_set(exp, &oinfo, ea, src_oa, oti, &set);
         if (rc)


@@ -2153,7 +2153,7 @@ static int lov_statfs_async(struct obd_device *obd, struct obd_info *oinfo,
         struct lov_obd *lov;
         int rc = 0;
         ENTRY;
         struct lov_obd *lov;
         int rc = 0;
         ENTRY;

-        
+

         LASSERT(oinfo != NULL);
         LASSERT(oinfo->oi_osfs != NULL);
 
         LASSERT(oinfo != NULL);
         LASSERT(oinfo->oi_osfs != NULL);
 


@@ -2188,38 +2188,28 @@ static int lov_statfs_async(struct obd_device *obd, struct obd_info *oinfo,
 }
 
 static int lov_statfs(struct obd_device *obd, struct obd_statfs *osfs,
 }
 
 static int lov_statfs(struct obd_device *obd, struct obd_statfs *osfs,

-                      __u64 max_age)
+                      __u64 max_age, __u32 flags)

 {
 {

-        struct lov_obd *lov = &obd->u.lov;
-        struct obd_statfs lov_sfs;
-        int set = 0;
-        int rc = 0, err;
-        int i;
+        struct ptlrpc_request_set *set = NULL;
+        struct obd_info oinfo = { { { 0 } } };
+        int rc = 0;

         ENTRY;
 
         ENTRY;
 

-        /* We only get block data from the OBD */
-        for (i = 0; i < lov->desc.ld_tgt_count; i++) {
-                if (!lov->lov_tgts[i] || !lov->lov_tgts[i]->ltd_active) {
-                        CDEBUG(D_HA, "lov idx %d inactive\n", i);
-                        continue;
-                }

 
 

-                err = obd_statfs(class_exp2obd(lov->lov_tgts[i]->ltd_exp),
-                                 &lov_sfs, max_age);
-                if (err) {
-                        if (lov->lov_tgts[i]->ltd_active && !rc)
-                                rc = err;
-                        continue;
-                }
-                lov_update_statfs(class_exp2obd(lov->lov_tgts[i]->ltd_exp),
-                                  osfs, &lov_sfs, set);
-                set++;
-        }
+        /* for obdclass we forbid using obd_statfs_rqset, but prefer using async
+         * statfs requests */
+        set = ptlrpc_prep_set();
+        if (set == NULL)
+                RETURN(-ENOMEM);

 
 

-        err = lov_fini_statfs(obd, osfs, set);
-        qos_update(lov);
+        oinfo.oi_osfs = osfs;
+        oinfo.oi_flags = flags;
+        rc = lov_statfs_async(obd, &oinfo, max_age, set);
+        if (rc == 0)
+                rc = ptlrpc_set_wait(set);
+        ptlrpc_set_destroy(set);

 
 

-        RETURN(rc ? rc : err);
+        RETURN(rc);

 }
 
 static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 }
 
 static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,


@@ -2243,19 +2233,20 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 
                 if ((index >= count))
                         RETURN(-ENODEV);
 
                 if ((index >= count))
                         RETURN(-ENODEV);

-                
+

                 if (!lov->lov_tgts[index])
                         /* Try again with the next index */
                         RETURN(-EAGAIN);
                 if (!lov->lov_tgts[index]->ltd_active)
                         RETURN(-ENODATA);
                 if (!lov->lov_tgts[index])
                         /* Try again with the next index */
                         RETURN(-EAGAIN);
                 if (!lov->lov_tgts[index]->ltd_active)
                         RETURN(-ENODATA);

-        
+

                 osc_obd = class_exp2obd(lov->lov_tgts[index]->ltd_exp);
                 if (!osc_obd)
                         RETURN(-EINVAL);
 
                 /* got statfs data */
                 osc_obd = class_exp2obd(lov->lov_tgts[index]->ltd_exp);
                 if (!osc_obd)
                         RETURN(-EINVAL);
 
                 /* got statfs data */

-                rc = obd_statfs(osc_obd, &stat_buf, cfs_time_current_64() - 1);
+                rc = obd_statfs(osc_obd, &stat_buf,
+                                cfs_time_current_64() - HZ, 0);

                 if (rc)
                         RETURN(rc);
                 if (copy_to_user(data->ioc_pbuf1, &stat_buf, data->ioc_plen1))
                 if (rc)
                         RETURN(rc);
                 if (copy_to_user(data->ioc_pbuf1, &stat_buf, data->ioc_plen1))

diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c
index ed93132..f1d65f4 100644 (file)
--- a/lustre/mdc/mdc_request.c
+++ b/lustre/mdc/mdc_request.c
@@ -1207,7 +1207,7 @@ int mdc_get_info(struct obd_export *exp, __u32 keylen, void *key,
 }
 
 static int mdc_statfs(struct obd_device *obd, struct obd_statfs *osfs,
 }
 
 static int mdc_statfs(struct obd_device *obd, struct obd_statfs *osfs,

-                      __u64 max_age)
+                      __u64 max_age, __u32 flags)

 {
         struct ptlrpc_request *req;
         struct obd_statfs     *msfs;
 {
         struct ptlrpc_request *req;
         struct obd_statfs     *msfs;


@@ -1221,6 +1221,12 @@ static int mdc_statfs(struct obd_device *obd, struct obd_statfs *osfs,
 
         ptlrpc_request_set_replen(req);
 
 
         ptlrpc_request_set_replen(req);
 

+        if (flags & OBD_STATFS_NODELAY) {
+                /* procfs requests not want stay in wait for avoid deadlock */
+                req->rq_no_resend = 1;
+                req->rq_no_delay = 1;
+        }
+

         rc = ptlrpc_queue_wait(req);
         if (rc) {
                 /* check connection error first */
         rc = ptlrpc_queue_wait(req);
         if (rc) {
                 /* check connection error first */

diff --git a/lustre/mds/handler.c b/lustre/mds/handler.c
index 0f3abd5..52bdf31 100644 (file)
--- a/lustre/mds/handler.c
+++ b/lustre/mds/handler.c
@@ -1069,7 +1069,7 @@ out_ucred:
 }
 
 static int mds_obd_statfs(struct obd_device *obd, struct obd_statfs *osfs,
 }
 
 static int mds_obd_statfs(struct obd_device *obd, struct obd_statfs *osfs,

-                          __u64 max_age)
+                          __u64 max_age, __u32 flags)

 {
         int rc;
 
 {
         int rc;
 


@@ -1103,7 +1103,7 @@ static int mds_statfs(struct ptlrpc_request *req)
         /* We call this so that we can cache a bit - 1 jiffie worth */
         rc = mds_obd_statfs(obd, lustre_msg_buf(req->rq_repmsg, REPLY_REC_OFF,
                                                 size[REPLY_REC_OFF]),
         /* We call this so that we can cache a bit - 1 jiffie worth */
         rc = mds_obd_statfs(obd, lustre_msg_buf(req->rq_repmsg, REPLY_REC_OFF,
                                                 size[REPLY_REC_OFF]),

-                            cfs_time_current_64() - HZ);
+                            cfs_time_current_64() - HZ, 0);

         if (rc) {
                 CERROR("mds_obd_statfs failed: rc %d\n", rc);
                 GOTO(out, rc);
         if (rc) {
                 CERROR("mds_obd_statfs failed: rc %d\n", rc);
                 GOTO(out, rc);

diff --git a/lustre/mds/lproc_mds.c b/lustre/mds/lproc_mds.c
index c535591..3e9dd72 100644 (file)
--- a/lustre/mds/lproc_mds.c
+++ b/lustre/mds/lproc_mds.c
@@ -105,13 +105,15 @@ static int lprocfs_mds_wr_evict_client(struct file *file, const char *buffer,
 
         obd_export_evict_by_nid(obd, tmpbuf+4);
 
 
         obd_export_evict_by_nid(obd, tmpbuf+4);
 

-        LPROCFS_ENTRY();
-        class_decref(obd);

 
         rc = ptlrpc_set_wait(set);
         if (rc)
                 CERROR("Failed to evict nid %s from OSTs: rc %d\n", tmpbuf + 4,
                        rc);
 
         rc = ptlrpc_set_wait(set);
         if (rc)
                 CERROR("Failed to evict nid %s from OSTs: rc %d\n", tmpbuf + 4,
                        rc);

+
+        LPROCFS_ENTRY();
+        class_decref(obd);
+

         ptlrpc_set_destroy(set);
         return count;
 }
         ptlrpc_set_destroy(set);
         return count;
 }

diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c
index 4fc61c9..3801243 100644 (file)
--- a/lustre/obdclass/lprocfs_status.c
+++ b/lustre/obdclass/lprocfs_status.c
@@ -441,7 +441,8 @@ int lprocfs_rd_blksize(char *page, char **start, off_t off, int count,
                        int *eof, void *data)
 {
         struct obd_statfs osfs;
                        int *eof, void *data)
 {
         struct obd_statfs osfs;

-        int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ);
+        int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ,
+                            OBD_STATFS_NODELAY);

         if (!rc) {
                 *eof = 1;
                 rc = snprintf(page, count, "%u\n", osfs.os_bsize);
         if (!rc) {
                 *eof = 1;
                 rc = snprintf(page, count, "%u\n", osfs.os_bsize);


@@ -453,7 +454,8 @@ int lprocfs_rd_kbytestotal(char *page, char **start, off_t off, int count,
                            int *eof, void *data)
 {
         struct obd_statfs osfs;
                            int *eof, void *data)
 {
         struct obd_statfs osfs;

-        int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ);
+        int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ,
+                            OBD_STATFS_NODELAY);

         if (!rc) {
                 __u32 blk_size = osfs.os_bsize >> 10;
                 __u64 result = osfs.os_blocks;
         if (!rc) {
                 __u32 blk_size = osfs.os_bsize >> 10;
                 __u64 result = osfs.os_blocks;


@@ -471,7 +473,8 @@ int lprocfs_rd_kbytesfree(char *page, char **start, off_t off, int count,
                           int *eof, void *data)
 {
         struct obd_statfs osfs;
                           int *eof, void *data)
 {
         struct obd_statfs osfs;

-        int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ);
+        int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ,
+                            OBD_STATFS_NODELAY);

         if (!rc) {
                 __u32 blk_size = osfs.os_bsize >> 10;
                 __u64 result = osfs.os_bfree;
         if (!rc) {
                 __u32 blk_size = osfs.os_bsize >> 10;
                 __u64 result = osfs.os_bfree;


@@ -489,7 +492,8 @@ int lprocfs_rd_kbytesavail(char *page, char **start, off_t off, int count,
                            int *eof, void *data)
 {
         struct obd_statfs osfs;
                            int *eof, void *data)
 {
         struct obd_statfs osfs;

-        int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ);
+        int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ,
+                            OBD_STATFS_NODELAY);

         if (!rc) {
                 __u32 blk_size = osfs.os_bsize >> 10;
                 __u64 result = osfs.os_bavail;
         if (!rc) {
                 __u32 blk_size = osfs.os_bsize >> 10;
                 __u64 result = osfs.os_bavail;


@@ -507,7 +511,8 @@ int lprocfs_rd_filestotal(char *page, char **start, off_t off, int count,
                           int *eof, void *data)
 {
         struct obd_statfs osfs;
                           int *eof, void *data)
 {
         struct obd_statfs osfs;

-        int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ);
+        int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ,
+                            OBD_STATFS_NODELAY);

         if (!rc) {
                 *eof = 1;
                 rc = snprintf(page, count, LPU64"\n", osfs.os_files);
         if (!rc) {
                 *eof = 1;
                 rc = snprintf(page, count, LPU64"\n", osfs.os_files);


@@ -520,7 +525,8 @@ int lprocfs_rd_filesfree(char *page, char **start, off_t off, int count,
                          int *eof, void *data)
 {
         struct obd_statfs osfs;
                          int *eof, void *data)
 {
         struct obd_statfs osfs;

-        int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ);
+        int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ,
+                            OBD_STATFS_NODELAY);

         if (!rc) {
                 *eof = 1;
                 rc = snprintf(page, count, LPU64"\n", osfs.os_ffree);
         if (!rc) {
                 *eof = 1;
                 rc = snprintf(page, count, LPU64"\n", osfs.os_ffree);

diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c
index efc01d2..b8b64ed 100644 (file)
--- a/lustre/obdfilter/filter.c
+++ b/lustre/obdfilter/filter.c
@@ -3330,7 +3330,7 @@ out:
 }
 
 static int filter_statfs(struct obd_device *obd, struct obd_statfs *osfs,
 }
 
 static int filter_statfs(struct obd_device *obd, struct obd_statfs *osfs,

-                         __u64 max_age)
+                         __u64 max_age, __u32 flags)

 {
         struct filter_obd *filter = &obd->u.filter;
         int blockbits = obd->u.obt.obt_sb->s_blocksize_bits;
 {
         struct filter_obd *filter = &obd->u.filter;
         int blockbits = obd->u.obt.obt_sb->s_blocksize_bits;


@@ -3426,7 +3426,7 @@ static int filter_precreate(struct obd_device *obd, struct obdo *oa,
                 OBD_ALLOC(osfs, sizeof(*osfs));
                 if (osfs == NULL)
                         RETURN(-ENOMEM);
                 OBD_ALLOC(osfs, sizeof(*osfs));
                 if (osfs == NULL)
                         RETURN(-ENOMEM);

-                rc = filter_statfs(obd, osfs, cfs_time_current_64() - HZ);
+                rc = filter_statfs(obd, osfs, cfs_time_current_64() - HZ, 0);

                 if (rc == 0 && osfs->os_bavail < (osfs->os_blocks >> 10)) {
                         CDEBUG(D_RPCTRACE,"%s: not enough space for create "
                                LPU64"\n", obd->obd_name, osfs->os_bavail <<
                 if (rc == 0 && osfs->os_bavail < (osfs->os_blocks >> 10)) {
                         CDEBUG(D_RPCTRACE,"%s: not enough space for create "
                                LPU64"\n", obd->obd_name, osfs->os_bavail <<

diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index cda2381..a5380b4 100644 (file)
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -3227,6 +3227,11 @@ static int osc_statfs_async(struct obd_device *obd, struct obd_info *oinfo,
         }
         ptlrpc_request_set_replen(req);
         req->rq_request_portal = OST_CREATE_PORTAL; //XXX FIXME bug 249
         }
         ptlrpc_request_set_replen(req);
         req->rq_request_portal = OST_CREATE_PORTAL; //XXX FIXME bug 249

+        if (oinfo->oi_flags & OBD_STATFS_NODELAY) {
+                /* procfs requests not want stat in wait for avoid deadlock */
+                req->rq_no_resend = 1;
+                req->rq_no_delay = 1;
+        }

 
         req->rq_interpret_reply = osc_statfs_interpret;
         CLASSERT (sizeof(*aa) <= sizeof(req->rq_async_args));
 
         req->rq_interpret_reply = osc_statfs_interpret;
         CLASSERT (sizeof(*aa) <= sizeof(req->rq_async_args));


@@ -3238,7 +3243,7 @@ static int osc_statfs_async(struct obd_device *obd, struct obd_info *oinfo,
 }
 
 static int osc_statfs(struct obd_device *obd, struct obd_statfs *osfs,
 }
 
 static int osc_statfs(struct obd_device *obd, struct obd_statfs *osfs,

-                      __u64 max_age)
+                      __u64 max_age, __u32 flags)

 {
         struct obd_statfs     *msfs;
         struct ptlrpc_request *req;
 {
         struct obd_statfs     *msfs;
         struct ptlrpc_request *req;


@@ -3263,6 +3268,12 @@ static int osc_statfs(struct obd_device *obd, struct obd_statfs *osfs,
         ptlrpc_request_set_replen(req);
         req->rq_request_portal = OST_CREATE_PORTAL; //XXX FIXME bug 249
 
         ptlrpc_request_set_replen(req);
         req->rq_request_portal = OST_CREATE_PORTAL; //XXX FIXME bug 249
 

+        if (flags & OBD_STATFS_NODELAY) {
+                /* procfs requests not want stat in wait for avoid deadlock */
+                req->rq_no_resend = 1;
+                req->rq_no_delay = 1;
+        }
+

         rc = ptlrpc_queue_wait(req);
         if (rc)
                 GOTO(out, rc);
         rc = ptlrpc_queue_wait(req);
         if (rc)
                 GOTO(out, rc);

diff --git a/lustre/ost/ost_handler.c b/lustre/ost/ost_handler.c
index e6a139c..02faf06 100644 (file)
--- a/lustre/ost/ost_handler.c
+++ b/lustre/ost/ost_handler.c
@@ -162,7 +162,7 @@ static int ost_statfs(struct ptlrpc_request *req)
         osfs = lustre_msg_buf(req->rq_repmsg, REPLY_REC_OFF, sizeof(*osfs));
 
         req->rq_status = obd_statfs(req->rq_export->exp_obd, osfs,
         osfs = lustre_msg_buf(req->rq_repmsg, REPLY_REC_OFF, sizeof(*osfs));
 
         req->rq_status = obd_statfs(req->rq_export->exp_obd, osfs,

-                                    cfs_time_current_64() - HZ);
+                                    cfs_time_current_64() - HZ, 0);

         if (OBD_FAIL_CHECK(OBD_FAIL_OST_ENOSPC))
                 osfs->os_bfree = osfs->os_bavail = 64;
         if (req->rq_status != 0)
         if (OBD_FAIL_CHECK(OBD_FAIL_OST_ENOSPC))
                 osfs->os_bfree = osfs->os_bavail = 64;
         if (req->rq_status != 0)

diff --git a/lustre/ptlrpc/lproc_ptlrpc.c b/lustre/ptlrpc/lproc_ptlrpc.c
index e4ffb22..c9b5347 100644 (file)
--- a/lustre/ptlrpc/lproc_ptlrpc.c
+++ b/lustre/ptlrpc/lproc_ptlrpc.c
@@ -564,6 +564,7 @@ int lprocfs_wr_ping(struct file *file, const char *buffer,
         ptlrpc_request_set_replen(req);
         req->rq_send_state = LUSTRE_IMP_FULL;
         req->rq_no_resend = 1;
         ptlrpc_request_set_replen(req);
         req->rq_send_state = LUSTRE_IMP_FULL;
         req->rq_no_resend = 1;

+        req->rq_no_delay = 1;

 
         rc = ptlrpc_queue_wait(req);
 
 
         rc = ptlrpc_queue_wait(req);