From e17cb17094bd72a17ad7b906489aca99d8a0b806 Mon Sep 17 00:00:00 2001 From: zhanghc Date: Thu, 2 Apr 2009 16:57:36 +0000 Subject: [PATCH] b=18364 1, fix a problem in osc_brw_async, which use the same obdo when there is too much data to be written into OST and trigger a LASSERT(!(oa->o_valid & bits)) in function "osc_announce_cached" 2, initialize the local variable "obdo" (struct obdo) in different functions, for it may cause some strange problems in system i=adilger@sun.com i=johann@sun.com --- lustre/include/lprocfs_status.h | 2 +- lustre/include/lustre/lustre_idl.h | 63 ++++++++++++++++++++++++------------ lustre/include/lustre_export.h | 2 +- lustre/include/lustre_quota.h | 2 +- lustre/liblustre/file.c | 2 +- lustre/liblustre/super.c | 2 +- lustre/llite/llite_close.c | 2 +- lustre/llite/rw.c | 4 +-- lustre/llite/rw26.c | 2 +- lustre/mdc/mdc_lib.c | 2 +- lustre/mds/mds_lov.c | 2 +- lustre/mgs/mgs_llog.c | 1 - lustre/obdecho/echo_client.c | 2 +- lustre/obdfilter/filter.c | 2 +- lustre/osc/osc_request.c | 65 ++++++++++++++++++++++++-------------- 15 files changed, 98 insertions(+), 57 deletions(-) diff --git a/lustre/include/lprocfs_status.h b/lustre/include/lprocfs_status.h index a80524d..03edd43 100644 --- a/lustre/include/lprocfs_status.h +++ b/lustre/include/lprocfs_status.h @@ -42,7 +42,6 @@ #ifndef _LPROCFS_SNMP_H #define _LPROCFS_SNMP_H -#include #if defined(__linux__) #include #elif defined(__APPLE__) @@ -52,6 +51,7 @@ #else #error Unsupported operating system. #endif +#include #undef LPROCFS #if (defined(__KERNEL__) && defined(CONFIG_PROC_FS)) diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index cd1d91f..b84ab24 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -448,26 +448,32 @@ typedef __u32 obd_gid; typedef __u32 obd_flag; typedef __u32 obd_count; -#define OBD_FL_INLINEDATA (0x00000001) -#define OBD_FL_OBDMDEXISTS (0x00000002) -#define OBD_FL_DELORPHAN (0x00000004) /* if set in o_flags delete orphans */ -#define OBD_FL_NORPC (0x00000008) /* set in o_flags do in OSC not OST */ -#define OBD_FL_IDONLY (0x00000010) /* set in o_flags only adjust obj id*/ -#define OBD_FL_RECREATE_OBJS (0x00000020) /* recreate missing obj */ -#define OBD_FL_DEBUG_CHECK (0x00000040) /* echo client/server debug check */ -#define OBD_FL_NO_USRQUOTA (0x00000100) /* the object's owner is over quota */ -#define OBD_FL_NO_GRPQUOTA (0x00000200) /* the object's group is over quota */ -#define OBD_FL_CREATE_CROW (0x00000400) /* object should be create on write */ -#define OBD_FL_TRUNCLOCK (0x00000800) /* delegate DLM locking during punch */ -#define OBD_FL_CKSUM_CRC32 (0x00001000) /* CRC32 checksum type */ -#define OBD_FL_CKSUM_ADLER (0x00002000) /* ADLER checksum type */ -#define OBD_FL_CKSUM_RESV1 (0x00004000) /* reserved for future checksum type */ -#define OBD_FL_CKSUM_RESV2 (0x00008000) /* reserved for future checksum type */ -#define OBD_FL_CKSUM_RESV3 (0x00010000) /* reserved for future checksum type */ -#define OBD_FL_SHRINK_GRANT (0x00020000) /* object shrink the grant */ - - -#define OBD_FL_CKSUM_ALL (OBD_FL_CKSUM_CRC32 | OBD_FL_CKSUM_ADLER) +enum obdo_flags { + OBD_FL_INLINEDATA = 0x00000001, + OBD_FL_OBDMDEXISTS = 0x00000002, + OBD_FL_DELORPHAN = 0x00000004, /* if set in o_flags delete orphans */ + OBD_FL_NORPC = 0x00000008, /* set in o_flags do in OSC not OST */ + OBD_FL_IDONLY = 0x00000010, /* set in o_flags only adjust obj id*/ + OBD_FL_RECREATE_OBJS= 0x00000020, /* recreate missing obj */ + OBD_FL_DEBUG_CHECK = 0x00000040, /* echo client/server debug check */ + OBD_FL_NO_USRQUOTA = 0x00000100, /* the object's owner is over quota */ + OBD_FL_NO_GRPQUOTA = 0x00000200, /* the object's group is over quota */ + OBD_FL_CREATE_CROW = 0x00000400, /* object should be create on write */ + OBD_FL_TRUNCLOCK = 0x00000800, /* delegate DLM locking during punch*/ + OBD_FL_CKSUM_CRC32 = 0x00001000, /* CRC32 checksum type */ + OBD_FL_CKSUM_ADLER = 0x00002000, /* ADLER checksum type */ + OBD_FL_CKSUM_RSVD1 = 0x00004000, /* for future cksum types */ + OBD_FL_CKSUM_RSVD2 = 0x00008000, /* for future cksum types */ + OBD_FL_CKSUM_RSVD3 = 0x00010000, /* for future cksum types */ + OBD_FL_SHRINK_GRANT = 0x00020000, /* object shrink the grant */ + + OBD_FL_CKSUM_ALL = OBD_FL_CKSUM_CRC32 | OBD_FL_CKSUM_ADLER, + + /* mask for local-only flag, which won't be sent over network */ + OBD_FL_LOCAL_MASK = 0xF0000000, + /* temporary OBDO used by osc_brw_async (see bug 18364) */ + OBD_FL_TEMPORARY = 0x10000000, +}; #define LOV_MAGIC_V1 0x0BD10BD0 #define LOV_MAGIC LOV_MAGIC_V1 @@ -1550,6 +1556,23 @@ struct obdo { #define o_dropped o_misc #define o_cksum o_nlink +static inline void lustre_set_wire_obdo(struct obdo *wobdo, struct obdo *lobdo) +{ + memcpy(wobdo, lobdo, sizeof(*lobdo)); + wobdo->o_flags &= ~OBD_FL_LOCAL_MASK; +} + +static inline void lustre_get_wire_obdo(struct obdo *lobdo, struct obdo *wobdo) +{ + obd_flag local_flags = lobdo->o_flags & OBD_FL_LOCAL_MASK; + + LASSERT(!(wobdo->o_flags & OBD_FL_LOCAL_MASK)); + + memcpy(lobdo, wobdo, sizeof(*lobdo)); + lobdo->o_flags &= ~OBD_FL_LOCAL_MASK; + lobdo->o_flags |= local_flags; +} + extern void lustre_swab_obdo (struct obdo *o); /* request structure for OST's */ diff --git a/lustre/include/lustre_export.h b/lustre/include/lustre_export.h index 47f389f..4d64595 100644 --- a/lustre/include/lustre_export.h +++ b/lustre/include/lustre_export.h @@ -37,8 +37,8 @@ #ifndef __EXPORT_H #define __EXPORT_H -#include #include +#include #include #include diff --git a/lustre/include/lustre_quota.h b/lustre/include/lustre_quota.h index 5871586..be073c4 100644 --- a/lustre/include/lustre_quota.h +++ b/lustre/include/lustre_quota.h @@ -47,8 +47,8 @@ #error Unsupported operating system. #endif -#include #include +#include #include #include #include diff --git a/lustre/liblustre/file.c b/lustre/liblustre/file.c index 94710cf..721318e 100644 --- a/lustre/liblustre/file.c +++ b/lustre/liblustre/file.c @@ -335,7 +335,7 @@ int llu_mdc_close(struct obd_export *mdc_exp, struct inode *inode) struct ll_file_data *fd = lli->lli_file_data; struct ptlrpc_request *req = NULL; struct obd_client_handle *och = &fd->fd_mds_och; - struct obdo obdo; + struct obdo obdo = { 0 }; int rc, valid; ENTRY; diff --git a/lustre/liblustre/super.c b/lustre/liblustre/super.c index 2b7fb4d..cac8c31 100644 --- a/lustre/liblustre/super.c +++ b/lustre/liblustre/super.c @@ -798,7 +798,7 @@ int llu_setattr_raw(struct inode *inode, struct iattr *attr) } } else if (ia_valid & (ATTR_MTIME | ATTR_MTIME_SET)) { struct obd_info oinfo = { { { 0 } } }; - struct obdo oa; + struct obdo oa = { 0 }; CDEBUG(D_INODE, "set mtime on OST inode %llu to %lu\n", (long long)st->st_ino, LTIME_S(attr->ia_mtime)); diff --git a/lustre/llite/llite_close.c b/lustre/llite/llite_close.c index f8a96d4..53fb413 100644 --- a/lustre/llite/llite_close.c +++ b/lustre/llite/llite_close.c @@ -138,7 +138,7 @@ static void ll_close_done_writing(struct inode *inode) struct ll_inode_info *lli = ll_i2info(inode); ldlm_policy_data_t policy = { .l_extent = {0, OBD_OBJECT_EOF } }; struct lustre_handle lockh = { 0 }; - struct obdo obdo; + struct obdo obdo = { 0 }; obd_flag valid; int rc, ast_flags = 0; ENTRY; diff --git a/lustre/llite/rw.c b/lustre/llite/rw.c index 9937764..aa33b2a 100644 --- a/lustre/llite/rw.c +++ b/lustre/llite/rw.c @@ -127,7 +127,7 @@ int ll_file_punch(struct inode * inode, loff_t new_size, int srvlock) { struct ll_inode_info *lli = ll_i2info(inode); struct obd_info oinfo = { { { 0 } } }; - struct obdo oa; + struct obdo oa = { 0 }; int rc; ENTRY; @@ -260,7 +260,7 @@ int ll_prepare_write(struct file *file, struct page *page, unsigned from, obd_off offset = ((obd_off)page->index) << CFS_PAGE_SHIFT; struct obd_info oinfo = { { { 0 } } }; struct brw_page pga; - struct obdo oa; + struct obdo oa = { 0 }; struct ost_lvb lvb; int rc = 0; ENTRY; diff --git a/lustre/llite/rw26.c b/lustre/llite/rw26.c index d957754..6815286 100644 --- a/lustre/llite/rw26.c +++ b/lustre/llite/rw26.c @@ -213,7 +213,7 @@ static ssize_t ll_direct_IO_26(int rw, struct kiocb *iocb, struct lov_stripe_md *lsm = lli->lli_smd; struct ptlrpc_request_set *set; struct obd_info oinfo; - struct obdo oa; + struct obdo oa = { 0 }; unsigned long seg; size_t size = MAX_DIO_SIZE; ENTRY; diff --git a/lustre/mdc/mdc_lib.c b/lustre/mdc/mdc_lib.c index f5106bd..c29b588 100644 --- a/lustre/mdc/mdc_lib.c +++ b/lustre/mdc/mdc_lib.c @@ -39,8 +39,8 @@ # include # include #endif -#include #include +#include #include "mdc_internal.h" #ifndef __KERNEL__ diff --git a/lustre/mds/mds_lov.c b/lustre/mds/mds_lov.c index 7809783..6c278e5 100644 --- a/lustre/mds/mds_lov.c +++ b/lustre/mds/mds_lov.c @@ -429,7 +429,7 @@ out: int mds_lov_clear_orphans(struct mds_obd *mds, struct obd_uuid *ost_uuid) { int rc; - struct obdo oa; + struct obdo oa = { 0 }; struct obd_trans_info oti = {0}; struct lov_stripe_md *empty_ea = NULL; ENTRY; diff --git a/lustre/mgs/mgs_llog.c b/lustre/mgs/mgs_llog.c index d24c42a..23b85e3 100644 --- a/lustre/mgs/mgs_llog.c +++ b/lustre/mgs/mgs_llog.c @@ -1437,7 +1437,6 @@ static int mgs_write_log_params(struct obd_device *obd, struct fs_db *fsdb, struct mgs_target_info *mti) { struct lustre_cfg_bufs bufs; - struct lustre_cfg *lcfg; char *logname; char *ptr = mti->mti_params; char *endptr, *tmp; diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c index 0a070d6..7f0ddb8 100644 --- a/lustre/obdecho/echo_client.c +++ b/lustre/obdecho/echo_client.c @@ -663,7 +663,7 @@ static void ec_ap_fill_obdo(void *data, int cmd, struct obdo *oa) { struct echo_async_page *eap = EAP_FROM_COOKIE(data); - memcpy(oa, &eap->eap_eas->eas_oa, sizeof(*oa)); + lustre_set_wire_obdo(oa, &eap->eap_eas->eas_oa); } static int ec_ap_completion(void *data, int cmd, struct obdo *oa, int rc) diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c index 01370ad..dccaab5 100644 --- a/lustre/obdfilter/filter.c +++ b/lustre/obdfilter/filter.c @@ -2830,7 +2830,7 @@ static int filter_unpackmd(struct obd_export *exp, struct lov_stripe_md **lsmp, static int filter_destroy_precreated(struct obd_export *exp, struct obdo *oa, struct filter_obd *filter) { - struct obdo doa; /* XXX obdo on stack */ + struct obdo doa = { 0 }; /* XXX obdo on stack */ obd_id last, id; int rc; ENTRY; diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index 7e9907f..5e9a7ea 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -176,7 +176,7 @@ static int osc_getattr_interpret(struct ptlrpc_request *req, lustre_swab_ost_body); if (body) { CDEBUG(D_INODE, "mode: %o\n", body->oa.o_mode); - memcpy(aa->aa_oi->oi_oa, &body->oa, sizeof(*aa->aa_oi->oi_oa)); + lustre_get_wire_obdo(aa->aa_oi->oi_oa, &body->oa); /* This should really be sent by the OST */ aa->aa_oi->oi_oa->o_blksize = PTLRPC_MAX_BRW_SIZE; @@ -206,7 +206,7 @@ static int osc_getattr_async(struct obd_export *exp, struct obd_info *oinfo, RETURN(-ENOMEM); body = lustre_msg_buf(req->rq_reqmsg, REQ_REC_OFF, sizeof(*body)); - memcpy(&body->oa, oinfo->oi_oa, sizeof(*oinfo->oi_oa)); + lustre_set_wire_obdo(&body->oa, oinfo->oi_oa); ptlrpc_req_set_repsize(req, 2, size); req->rq_interpret_reply = osc_getattr_interpret; @@ -233,7 +233,7 @@ static int osc_getattr(struct obd_export *exp, struct obd_info *oinfo) RETURN(-ENOMEM); body = lustre_msg_buf(req->rq_reqmsg, REQ_REC_OFF, sizeof(*body)); - memcpy(&body->oa, oinfo->oi_oa, sizeof(*oinfo->oi_oa)); + lustre_set_wire_obdo(&body->oa, oinfo->oi_oa); ptlrpc_req_set_repsize(req, 2, size); @@ -251,7 +251,7 @@ static int osc_getattr(struct obd_export *exp, struct obd_info *oinfo) } CDEBUG(D_INODE, "mode: %o\n", body->oa.o_mode); - memcpy(oinfo->oi_oa, &body->oa, sizeof(*oinfo->oi_oa)); + lustre_get_wire_obdo(oinfo->oi_oa, &body->oa); /* This should really be sent by the OST */ oinfo->oi_oa->o_blksize = PTLRPC_MAX_BRW_SIZE; @@ -278,7 +278,7 @@ static int osc_setattr(struct obd_export *exp, struct obd_info *oinfo, RETURN(-ENOMEM); body = lustre_msg_buf(req->rq_reqmsg, REQ_REC_OFF, sizeof(*body)); - memcpy(&body->oa, oinfo->oi_oa, sizeof(*oinfo->oi_oa)); + lustre_set_wire_obdo(&body->oa, oinfo->oi_oa); ptlrpc_req_set_repsize(req, 2, size); @@ -291,7 +291,7 @@ static int osc_setattr(struct obd_export *exp, struct obd_info *oinfo, if (body == NULL) GOTO(out, rc = -EPROTO); - memcpy(oinfo->oi_oa, &body->oa, sizeof(*oinfo->oi_oa)); + lustre_get_wire_obdo(oinfo->oi_oa, &body->oa); EXIT; out: @@ -315,7 +315,7 @@ static int osc_setattr_interpret(struct ptlrpc_request *req, GOTO(out, rc = -EPROTO); } - memcpy(aa->aa_oi->oi_oa, &body->oa, sizeof(*aa->aa_oi->oi_oa)); + lustre_get_wire_obdo(aa->aa_oi->oi_oa, &body->oa); out: rc = aa->aa_oi->oi_cb_up(aa->aa_oi, rc); RETURN(rc); @@ -343,7 +343,7 @@ static int osc_setattr_async(struct obd_export *exp, struct obd_info *oinfo, oinfo->oi_oa->o_lcookie = *oti->oti_logcookies; } - memcpy(&body->oa, oinfo->oi_oa, sizeof(*oinfo->oi_oa)); + lustre_set_wire_obdo(&body->oa, oinfo->oi_oa); ptlrpc_req_set_repsize(req, 2, size); /* do mds to ost setattr asynchronouly */ if (!rqset) { @@ -388,7 +388,7 @@ int osc_real_create(struct obd_export *exp, struct obdo *oa, GOTO(out, rc = -ENOMEM); body = lustre_msg_buf(req->rq_reqmsg, REQ_REC_OFF, sizeof(*body)); - memcpy(&body->oa, oa, sizeof(body->oa)); + lustre_set_wire_obdo(&body->oa, oa); ptlrpc_req_set_repsize(req, 2, size); if ((oa->o_valid & OBD_MD_FLFLAGS) && @@ -410,7 +410,7 @@ int osc_real_create(struct obd_export *exp, struct obdo *oa, GOTO (out_req, rc = -EPROTO); } - memcpy(oa, &body->oa, sizeof(*oa)); + lustre_get_wire_obdo(oa, &body->oa); /* This should really be sent by the OST */ oa->o_blksize = PTLRPC_MAX_BRW_SIZE; @@ -459,7 +459,7 @@ static int osc_punch_interpret(struct ptlrpc_request *req, GOTO(out, rc = -EPROTO); } - memcpy(aa->aa_oi->oi_oa, &body->oa, sizeof(*aa->aa_oi->oi_oa)); + lustre_get_wire_obdo(aa->aa_oi->oi_oa, &body->oa); out: rc = aa->aa_oi->oi_cb_up(aa->aa_oi, rc); RETURN(rc); @@ -489,7 +489,7 @@ static int osc_punch(struct obd_export *exp, struct obd_info *oinfo, ptlrpc_at_set_req_timeout(req); body = lustre_msg_buf(req->rq_reqmsg, REQ_REC_OFF, sizeof(*body)); - memcpy(&body->oa, oinfo->oi_oa, sizeof(*oinfo->oi_oa)); + lustre_set_wire_obdo(&body->oa, oinfo->oi_oa); /* overload the size and blocks fields in the oa with start/end */ body->oa.o_size = oinfo->oi_policy.l_extent.start; @@ -527,7 +527,7 @@ static int osc_sync(struct obd_export *exp, struct obdo *oa, RETURN(-ENOMEM); body = lustre_msg_buf(req->rq_reqmsg, REQ_REC_OFF, sizeof(*body)); - memcpy(&body->oa, oa, sizeof(*oa)); + lustre_set_wire_obdo(&body->oa, oa); /* overload the size and blocks fields in the oa with start/end */ body->oa.o_size = start; @@ -547,7 +547,7 @@ static int osc_sync(struct obd_export *exp, struct obdo *oa, GOTO (out, rc = -EPROTO); } - memcpy(oa, &body->oa, sizeof(*oa)); + lustre_get_wire_obdo(oa, &body->oa); EXIT; out: @@ -653,7 +653,7 @@ static int osc_destroy(struct obd_export *exp, struct obdo *oa, oa->o_lcookie = *oti->oti_logcookies; } - memcpy(&body->oa, oa, sizeof(*oa)); + lustre_set_wire_obdo(&body->oa, oa); ptlrpc_req_set_repsize(req, 2, size); /* don't throttle destroy RPCs for the MDT */ @@ -1138,8 +1138,7 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,struct obdo *oa, niobuf = lustre_msg_buf(req->rq_reqmsg, REQ_REC_OFF + 2, niocount * sizeof(*niobuf)); - memcpy(&body->oa, oa, sizeof(*oa)); - + lustre_set_wire_obdo(&body->oa, oa); obdo_to_ioobj(oa, ioobj); ioobj->ioo_bufcnt = niocount; @@ -1200,8 +1199,10 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,struct obdo *oa, * it can be changed via lprocfs */ cksum_type_t cksum_type = cli->cl_cksum_type; - if ((body->oa.o_valid & OBD_MD_FLFLAGS) == 0) - oa->o_flags = body->oa.o_flags = 0; + if ((body->oa.o_valid & OBD_MD_FLFLAGS) == 0) { + oa->o_flags &= OBD_FL_LOCAL_MASK; + body->oa.o_flags = 0; + } body->oa.o_flags |= cksum_type_pack(cksum_type); body->oa.o_valid |= OBD_MD_FLCKSUM | OBD_MD_FLFLAGS; body->oa.o_cksum = osc_checksum_bulk(requested_nob, @@ -1444,7 +1445,7 @@ static int osc_brw_fini_request(struct ptlrpc_request *req, int rc) } out: if (rc >= 0) - memcpy(aa->aa_oa, &body->oa, sizeof(*aa->aa_oa)); + lustre_get_wire_obdo(aa->aa_oa, &body->oa); RETURN(rc); } @@ -1802,6 +1803,7 @@ static int osc_brw_async(int cmd, struct obd_export *exp, sort_brw_pages(ppga, page_count); while (page_count) { struct brw_page **copy; + struct obdo *oa; obd_count pages_per_brw; pages_per_brw = min_t(obd_count, page_count, @@ -1815,15 +1817,29 @@ static int osc_brw_async(int cmd, struct obd_export *exp, if (copy == NULL) GOTO(out, rc = -ENOMEM); memcpy(copy, ppga, pages_per_brw * sizeof(*copy)); - } else + + OBDO_ALLOC(oa); + if (oa == NULL) { + OBD_FREE(copy, pages_per_brw * sizeof(*copy)); + GOTO(out, rc = -ENOMEM); + } + memcpy(oa, oinfo->oi_oa, sizeof(*oa)); + oa->o_flags |= OBD_FL_TEMPORARY; + } else { copy = ppga; + oa = oinfo->oi_oa; + LASSERT(!(oa->o_flags & OBD_FL_TEMPORARY)); + } - rc = async_internal(cmd, exp, oinfo->oi_oa, oinfo->oi_md, - pages_per_brw, copy, set); + rc = async_internal(cmd, exp, oa, oinfo->oi_md, pages_per_brw, + copy, set); if (rc != 0) { if (copy != ppga) OBD_FREE(copy, pages_per_brw * sizeof(*copy)); + + if (oa->o_flags & OBD_FL_TEMPORARY) + OBDO_FREE(oa); break; } @@ -2143,6 +2159,9 @@ static int brw_interpret(struct ptlrpc_request *request, void *data, int rc) obd_count i; for (i = 0; i < aa->aa_page_count; i++) osc_release_write_grant(aa->aa_cli, aa->aa_ppga[i], 1); + + if (aa->aa_oa->o_flags & OBD_FL_TEMPORARY) + OBDO_FREE(aa->aa_oa); } osc_wake_cache_waiters(cli); osc_check_rpcs(cli); -- 1.8.3.1