obd objects and this also block reconnect with -EALREADY.
This isn't fix all lprocfs bugs - but make it rare.
b=14533
i=adilger
i=nathan
* RHEL 4 and RHEL 5/SLES 10 clients behaves differently on 'cd' to a
removed cwd "./" (refer to Bugzilla 14399).
+Severity : normal
+Bugzilla : 14533
+Frequency : rare, on recovery
+Description: read procfs can produce deadlock in some situation
+Details : Holding lprocfs lock which send rpc can produce block for destroy
+ obd objects and this also block reconnect with -EALREADY. This isn't
+ fix all lprocfs bugs - but make it rare.
+
Severity : enhancement
Bugzilla : 15152
Description: Update kernel to RHEL5 2.6.18-53.1.14.el5.
};
extern void lustre_swab_obd_statfs (struct obd_statfs *os);
+#define OBD_STATFS_NODELAY 0x0001 /* requests should be send without delay
+ * and resends for avoid deadlocks */
/* ost_body.data values for OST_BRW */
* OSC. (e.g. lov_prep_enqueue_set initialises extent of the policy,
* and osc_enqueue passes it into ldlm_lock_match & ldlm_cli_enqueue. */
ldlm_policy_data_t oi_policy;
- /* Flags used while lock handling. The flags obtained on the enqueue
- * request are set here, therefore they are request specific. */
+ /* Flags used for set request specific flags:
+ - while lock handling, the flags obtained on the enqueue
+ request are set here.
+ - while stats, the flags used for control delay/resend.
+ */
int oi_flags;
/* Lock handle specific for every OSC lock. */
struct lustre_handle *oi_lockh;
int (*o_fid_delete)(struct obd_export *exp, const struct lu_fid *fid);
int (*o_statfs)(struct obd_device *obd, struct obd_statfs *osfs,
- __u64 max_age);
+ __u64 max_age, __u32 flags);
int (*o_statfs_async)(struct obd_device *obd, struct obd_info *oinfo,
__u64 max_age, struct ptlrpc_request_set *set);
int (*o_packmd)(struct obd_export *exp, struct lov_mds_md **disk_tgt,
static inline int obd_statfs_rqset(struct obd_device *obd,
struct obd_statfs *osfs, __u64 max_age,
- int quick_pry)
+ __u32 flags)
{
struct ptlrpc_request_set *set = NULL;
struct obd_info oinfo = { { { 0 } } };
RETURN(-ENOMEM);
oinfo.oi_osfs = osfs;
+ oinfo.oi_flags = flags;
rc = obd_statfs_async(obd, &oinfo, max_age, set);
- if (rc == 0) {
- struct ptlrpc_request *req;
-
- if (quick_pry)
- list_for_each_entry(req, &set->set_requests,
- rq_set_chain) {
- spin_lock(&req->rq_lock);
- req->rq_no_resend = 1;
- req->rq_no_delay = 1;
- spin_unlock(&req->rq_lock);
- }
+ if (rc == 0)
rc = ptlrpc_set_wait(set);
- }
ptlrpc_set_destroy(set);
RETURN(rc);
}
* If the cache is older than @max_age we will get a new value from the
* target. Use a value of "cfs_time_current() + HZ" to guarantee freshness. */
static inline int obd_statfs(struct obd_device *obd, struct obd_statfs *osfs,
- __u64 max_age)
+ __u64 max_age, __u32 flags)
{
int rc = 0;
ENTRY;
CDEBUG(D_SUPER, "osfs "LPU64", max_age "LPU64"\n",
obd->obd_osfs_age, max_age);
if (cfs_time_before_64(obd->obd_osfs_age, max_age)) {
- rc = OBP(obd, statfs)(obd, osfs, max_age);
+ rc = OBP(obd, statfs)(obd, osfs, max_age, flags);
if (rc == 0) {
spin_lock(&obd->obd_osfs_lock);
memcpy(&obd->obd_osfs, osfs, sizeof(obd->obd_osfs));
if (!ns)
RETURN(ELDLM_OK);
+#ifdef LPROCFS
+ {
+ struct proc_dir_entry *dir;
+ dir = lprocfs_srch(ldlm_ns_proc_dir, ns->ns_name);
+ if (dir == NULL) {
+ CERROR("dlm namespace %s has no procfs dir?\n",
+ ns->ns_name);
+ } else {
+ lprocfs_remove(&dir);
+ }
+ }
+#endif
+
mutex_down(ldlm_namespace_lock(ns->ns_client));
list_del(&ns->ns_list_chain);
atomic_dec(ldlm_namespace_nr(ns->ns_client));
if (!ns)
RETURN(ELDLM_OK);
-#ifdef LPROCFS
- {
- struct proc_dir_entry *dir;
- dir = lprocfs_srch(ldlm_ns_proc_dir, ns->ns_name);
- if (dir == NULL) {
- CERROR("dlm namespace %s has no procfs dir?\n",
- ns->ns_name);
- } else {
- lprocfs_remove(&dir);
- }
- }
-#endif
OBD_VFREE(ns->ns_hash, sizeof(*ns->ns_hash) * RES_HASH_SIZE);
OBD_FREE(ns->ns_name, strlen(ns->ns_name) + 1);
OBD_FREE_PTR(ns);
int rc;
ENTRY;
- rc = obd_statfs(class_exp2obd(sbi->ll_md_exp), osfs, max_age);
+ rc = obd_statfs(class_exp2obd(sbi->ll_md_exp), osfs, max_age, 0);
if (rc) {
CERROR("md_statfs fails: rc = %d\n", rc);
RETURN(rc);
}
sbi->ll_md_exp = class_conn2export(&md_conn);
- err = obd_statfs(obd, &osfs, 100000000);
+ err = obd_statfs(obd, &osfs, 100000000, 0);
if (err)
GOTO(out_md, err);
int ll_statfs(struct dentry *de, struct kstatfs *sfs);
#endif
int ll_statfs_internal(struct super_block *sb, struct obd_statfs *osfs,
- __u64 max_age);
+ __u64 max_age, __u32 flags);
void ll_update_inode(struct inode *inode, struct lustre_md *md);
void ll_read_inode2(struct inode *inode, void *opaque);
void ll_delete_inode(struct inode *inode);
GOTO(out_md, err);
}
- err = obd_statfs(obd, &osfs, cfs_time_current_64() - HZ);
+ err = obd_statfs(obd, &osfs, cfs_time_current_64() - HZ, 0);
if (err)
GOTO(out_md_fid, err);
}
int ll_statfs_internal(struct super_block *sb, struct obd_statfs *osfs,
- __u64 max_age)
+ __u64 max_age, __u32 flags)
{
struct ll_sb_info *sbi = ll_s2sbi(sb);
struct obd_statfs obd_osfs;
int rc;
ENTRY;
- rc = obd_statfs(class_exp2obd(sbi->ll_md_exp), osfs, max_age);
+ rc = obd_statfs(class_exp2obd(sbi->ll_md_exp), osfs, max_age, flags);
if (rc) {
CERROR("md_statfs fails: rc = %d\n", rc);
RETURN(rc);
osfs->os_bavail, osfs->os_blocks, osfs->os_ffree,osfs->os_files);
rc = obd_statfs_rqset(class_exp2obd(sbi->ll_dt_exp),
- &obd_osfs, max_age, 0);
+ &obd_osfs, max_age, flags);
if (rc) {
CERROR("obd_statfs fails: rc = %d\n", rc);
RETURN(rc);
/* For now we will always get up-to-date statfs values, but in the
* future we may allow some amount of caching on the client (e.g.
* from QOS or lprocfs updates). */
- rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - 1);
+ rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - 1, 0);
if (rc)
return rc;
int rc;
LASSERT(sb != NULL);
- rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ);
+ rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ,
+ OBD_STATFS_NODELAY);
if (!rc) {
*eof = 1;
rc = snprintf(page, count, "%u\n", osfs.os_bsize);
int rc;
LASSERT(sb != NULL);
- rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ);
+ rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ,
+ OBD_STATFS_NODELAY);
if (!rc) {
__u32 blk_size = osfs.os_bsize >> 10;
__u64 result = osfs.os_blocks;
int rc;
LASSERT(sb != NULL);
- rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ);
+ rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ,
+ OBD_STATFS_NODELAY);
if (!rc) {
__u32 blk_size = osfs.os_bsize >> 10;
__u64 result = osfs.os_bfree;
int rc;
LASSERT(sb != NULL);
- rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ);
+ rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ,
+ OBD_STATFS_NODELAY);
if (!rc) {
__u32 blk_size = osfs.os_bsize >> 10;
__u64 result = osfs.os_bavail;
int rc;
LASSERT(sb != NULL);
- rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ);
+ rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ,
+ OBD_STATFS_NODELAY);
if (!rc) {
*eof = 1;
rc = snprintf(page, count, LPU64"\n", osfs.os_files);
int rc;
LASSERT(sb != NULL);
- rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ);
+ rc = ll_statfs_internal(sb, &osfs, cfs_time_current_64() - HZ,
+ OBD_STATFS_NODELAY);
if (!rc) {
*eof = 1;
rc = snprintf(page, count, LPU64"\n", osfs.os_ffree);
if ((index >= lmv->desc.ld_tgt_count))
RETURN(-ENODEV);
-
+
if (!lmv->tgts[index].ltd_active)
RETURN(-ENODATA);
RETURN(-EINVAL);
/* got statfs data */
- rc = obd_statfs(mdc_obd, &stat_buf, cfs_time_current_64() - 1);
+ rc = obd_statfs(mdc_obd, &stat_buf,
+ cfs_time_current_64() - HZ, 0);
if (rc)
RETURN(rc);
if (copy_to_user(data->ioc_pbuf1, &stat_buf, data->ioc_plen1))
}
static int lmv_statfs(struct obd_device *obd, struct obd_statfs *osfs,
- __u64 max_age)
+ __u64 max_age, __u32 flags)
{
struct lmv_obd *lmv = &obd->u.lmv;
struct obd_statfs *temp;
if (lmv->tgts[i].ltd_exp == NULL)
continue;
- rc = obd_statfs(lmv->tgts[i].ltd_exp->exp_obd, temp, max_age);
+ rc = obd_statfs(lmv->tgts[i].ltd_exp->exp_obd, temp,
+ max_age, flags);
if (rc) {
CERROR("can't stat MDS #%d (%s), error %d\n", i,
lmv->tgts[i].ltd_exp->exp_obd->obd_name,
}
maxage = cfs_time_shift_64(-lov->desc.ld_qos_maxage);
- obd_statfs_rqset(exp->exp_obd, &osfs, maxage, 1);
+ obd_statfs_rqset(exp->exp_obd, &osfs, maxage, OBD_STATFS_NODELAY);
rc = lov_prep_create_set(exp, &oinfo, ea, src_oa, oti, &set);
if (rc)
struct lov_obd *lov;
int rc = 0;
ENTRY;
-
+
LASSERT(oinfo != NULL);
LASSERT(oinfo->oi_osfs != NULL);
}
static int lov_statfs(struct obd_device *obd, struct obd_statfs *osfs,
- __u64 max_age)
+ __u64 max_age, __u32 flags)
{
- struct lov_obd *lov = &obd->u.lov;
- struct obd_statfs lov_sfs;
- int set = 0;
- int rc = 0, err;
- int i;
+ struct ptlrpc_request_set *set = NULL;
+ struct obd_info oinfo = { { { 0 } } };
+ int rc = 0;
ENTRY;
- /* We only get block data from the OBD */
- for (i = 0; i < lov->desc.ld_tgt_count; i++) {
- if (!lov->lov_tgts[i] || !lov->lov_tgts[i]->ltd_active) {
- CDEBUG(D_HA, "lov idx %d inactive\n", i);
- continue;
- }
- err = obd_statfs(class_exp2obd(lov->lov_tgts[i]->ltd_exp),
- &lov_sfs, max_age);
- if (err) {
- if (lov->lov_tgts[i]->ltd_active && !rc)
- rc = err;
- continue;
- }
- lov_update_statfs(class_exp2obd(lov->lov_tgts[i]->ltd_exp),
- osfs, &lov_sfs, set);
- set++;
- }
+ /* for obdclass we forbid using obd_statfs_rqset, but prefer using async
+ * statfs requests */
+ set = ptlrpc_prep_set();
+ if (set == NULL)
+ RETURN(-ENOMEM);
- err = lov_fini_statfs(obd, osfs, set);
- qos_update(lov);
+ oinfo.oi_osfs = osfs;
+ oinfo.oi_flags = flags;
+ rc = lov_statfs_async(obd, &oinfo, max_age, set);
+ if (rc == 0)
+ rc = ptlrpc_set_wait(set);
+ ptlrpc_set_destroy(set);
- RETURN(rc ? rc : err);
+ RETURN(rc);
}
static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
if ((index >= count))
RETURN(-ENODEV);
-
+
if (!lov->lov_tgts[index])
/* Try again with the next index */
RETURN(-EAGAIN);
if (!lov->lov_tgts[index]->ltd_active)
RETURN(-ENODATA);
-
+
osc_obd = class_exp2obd(lov->lov_tgts[index]->ltd_exp);
if (!osc_obd)
RETURN(-EINVAL);
/* got statfs data */
- rc = obd_statfs(osc_obd, &stat_buf, cfs_time_current_64() - 1);
+ rc = obd_statfs(osc_obd, &stat_buf,
+ cfs_time_current_64() - HZ, 0);
if (rc)
RETURN(rc);
if (copy_to_user(data->ioc_pbuf1, &stat_buf, data->ioc_plen1))
}
static int mdc_statfs(struct obd_device *obd, struct obd_statfs *osfs,
- __u64 max_age)
+ __u64 max_age, __u32 flags)
{
struct ptlrpc_request *req;
struct obd_statfs *msfs;
ptlrpc_request_set_replen(req);
+ if (flags & OBD_STATFS_NODELAY) {
+ /* procfs requests not want stay in wait for avoid deadlock */
+ req->rq_no_resend = 1;
+ req->rq_no_delay = 1;
+ }
+
rc = ptlrpc_queue_wait(req);
if (rc) {
/* check connection error first */
}
static int mds_obd_statfs(struct obd_device *obd, struct obd_statfs *osfs,
- __u64 max_age)
+ __u64 max_age, __u32 flags)
{
int rc;
/* We call this so that we can cache a bit - 1 jiffie worth */
rc = mds_obd_statfs(obd, lustre_msg_buf(req->rq_repmsg, REPLY_REC_OFF,
size[REPLY_REC_OFF]),
- cfs_time_current_64() - HZ);
+ cfs_time_current_64() - HZ, 0);
if (rc) {
CERROR("mds_obd_statfs failed: rc %d\n", rc);
GOTO(out, rc);
obd_export_evict_by_nid(obd, tmpbuf+4);
- LPROCFS_ENTRY();
- class_decref(obd);
rc = ptlrpc_set_wait(set);
if (rc)
CERROR("Failed to evict nid %s from OSTs: rc %d\n", tmpbuf + 4,
rc);
+
+ LPROCFS_ENTRY();
+ class_decref(obd);
+
ptlrpc_set_destroy(set);
return count;
}
int *eof, void *data)
{
struct obd_statfs osfs;
- int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ);
+ int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ,
+ OBD_STATFS_NODELAY);
if (!rc) {
*eof = 1;
rc = snprintf(page, count, "%u\n", osfs.os_bsize);
int *eof, void *data)
{
struct obd_statfs osfs;
- int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ);
+ int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ,
+ OBD_STATFS_NODELAY);
if (!rc) {
__u32 blk_size = osfs.os_bsize >> 10;
__u64 result = osfs.os_blocks;
int *eof, void *data)
{
struct obd_statfs osfs;
- int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ);
+ int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ,
+ OBD_STATFS_NODELAY);
if (!rc) {
__u32 blk_size = osfs.os_bsize >> 10;
__u64 result = osfs.os_bfree;
int *eof, void *data)
{
struct obd_statfs osfs;
- int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ);
+ int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ,
+ OBD_STATFS_NODELAY);
if (!rc) {
__u32 blk_size = osfs.os_bsize >> 10;
__u64 result = osfs.os_bavail;
int *eof, void *data)
{
struct obd_statfs osfs;
- int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ);
+ int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ,
+ OBD_STATFS_NODELAY);
if (!rc) {
*eof = 1;
rc = snprintf(page, count, LPU64"\n", osfs.os_files);
int *eof, void *data)
{
struct obd_statfs osfs;
- int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ);
+ int rc = obd_statfs(data, &osfs, cfs_time_current_64() - HZ,
+ OBD_STATFS_NODELAY);
if (!rc) {
*eof = 1;
rc = snprintf(page, count, LPU64"\n", osfs.os_ffree);
}
static int filter_statfs(struct obd_device *obd, struct obd_statfs *osfs,
- __u64 max_age)
+ __u64 max_age, __u32 flags)
{
struct filter_obd *filter = &obd->u.filter;
int blockbits = obd->u.obt.obt_sb->s_blocksize_bits;
OBD_ALLOC(osfs, sizeof(*osfs));
if (osfs == NULL)
RETURN(-ENOMEM);
- rc = filter_statfs(obd, osfs, cfs_time_current_64() - HZ);
+ rc = filter_statfs(obd, osfs, cfs_time_current_64() - HZ, 0);
if (rc == 0 && osfs->os_bavail < (osfs->os_blocks >> 10)) {
CDEBUG(D_RPCTRACE,"%s: not enough space for create "
LPU64"\n", obd->obd_name, osfs->os_bavail <<
}
ptlrpc_request_set_replen(req);
req->rq_request_portal = OST_CREATE_PORTAL; //XXX FIXME bug 249
+ if (oinfo->oi_flags & OBD_STATFS_NODELAY) {
+ /* procfs requests not want stat in wait for avoid deadlock */
+ req->rq_no_resend = 1;
+ req->rq_no_delay = 1;
+ }
req->rq_interpret_reply = osc_statfs_interpret;
CLASSERT (sizeof(*aa) <= sizeof(req->rq_async_args));
}
static int osc_statfs(struct obd_device *obd, struct obd_statfs *osfs,
- __u64 max_age)
+ __u64 max_age, __u32 flags)
{
struct obd_statfs *msfs;
struct ptlrpc_request *req;
ptlrpc_request_set_replen(req);
req->rq_request_portal = OST_CREATE_PORTAL; //XXX FIXME bug 249
+ if (flags & OBD_STATFS_NODELAY) {
+ /* procfs requests not want stat in wait for avoid deadlock */
+ req->rq_no_resend = 1;
+ req->rq_no_delay = 1;
+ }
+
rc = ptlrpc_queue_wait(req);
if (rc)
GOTO(out, rc);
osfs = lustre_msg_buf(req->rq_repmsg, REPLY_REC_OFF, sizeof(*osfs));
req->rq_status = obd_statfs(req->rq_export->exp_obd, osfs,
- cfs_time_current_64() - HZ);
+ cfs_time_current_64() - HZ, 0);
if (OBD_FAIL_CHECK(OBD_FAIL_OST_ENOSPC))
osfs->os_bfree = osfs->os_bavail = 64;
if (req->rq_status != 0)
ptlrpc_request_set_replen(req);
req->rq_send_state = LUSTRE_IMP_FULL;
req->rq_no_resend = 1;
+ req->rq_no_delay = 1;
rc = ptlrpc_queue_wait(req);