From: nikita Date: Fri, 5 Aug 2005 18:12:10 +0000 (+0000) Subject: comment for ll_readahead_state and assertion in osc_send_oap_request() from b1_4 X-Git-Tag: v1_7_100~996 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=645ffb9d3cdc02e3b025dbe420b92a7cf75d59c5 comment for ll_readahead_state and assertion in osc_send_oap_request() from b1_4 --- diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h index d0bca7a..8a58404 100644 --- a/lustre/llite/llite_internal.h +++ b/lustre/llite/llite_internal.h @@ -49,7 +49,7 @@ struct ll_sb_info { /* this protects pglist and max_r_a_pages. It isn't safe to grab from * interrupt contexts. */ spinlock_t ll_lock; - + struct obd_uuid ll_sb_uuid; struct obd_export *ll_md_exp; struct obd_export *ll_dt_exp; @@ -125,10 +125,45 @@ struct ll_gns_ctl { #define LL_GNS_UMOUNT (1 << 0) #define LL_GNS_CHECK (1 << 1) +/* + * per file-descriptor read-ahead data. + */ struct ll_readahead_state { spinlock_t ras_lock; - unsigned long ras_last_readpage, ras_consecutive; + /* + * index of the last page that read(2) needed and that wasn't in the + * cache. Used by ras_update() to detect seeks. + * + * XXX nikita: if access seeks into cached region, Lustre doesn't see + * this. + */ + unsigned long ras_last_readpage; + /* + * number of pages read after last read-ahead window reset. As window + * is reset on each seek, this is effectively a number of consecutive + * accesses. Maybe ->ras_accessed_in_window is better name. + * + * XXX nikita: window is also reset (by ras_update()) when Lustre + * believes that memory pressure evicts read-ahead pages. In that + * case, it probably doesn't make sense to expand window to + * PTLRPC_MAX_BRW_PAGES on the third access. + */ + unsigned long ras_consecutive; + /* + * Parameters of current read-ahead window. Handled by + * ras_update(). On the initial access to the file or after a seek, + * window is reset to 0. After 3 consecutive accesses, window is + * expanded to PTLRPC_MAX_BRW_PAGES. Afterwards, window is enlarged by + * PTLRPC_MAX_BRW_PAGES chunks up to ->ra_max_pages. + */ unsigned long ras_window_start, ras_window_len; + /* + * Where next read-ahead should start at. This lies within read-ahead + * window. Read-ahead window is read in pieces rather than at once + * because: 1. lustre limits total number of pages under read-ahead by + * ->ra_max_pages (see ll_ra_count_get()), 2. client cannot read pages + * not covered by DLM lock. + */ unsigned long ras_next_readahead; }; @@ -241,7 +276,7 @@ extern struct file_operations ll_dir_operations; extern struct inode_operations ll_dir_inode_operations; /* llite/namei.c */ -int ll_objects_destroy(struct ptlrpc_request *request, +int ll_objects_destroy(struct ptlrpc_request *request, struct inode *dir, int offset); struct inode *ll_iget(struct super_block *sb, ino_t hash, struct lustre_md *lic); @@ -277,7 +312,7 @@ int ll_getxattr(struct dentry *, const char *, void *, size_t); int ll_listxattr(struct dentry *, char *, size_t); int ll_removexattr(struct dentry *, const char *); extern int ll_inode_permission(struct inode *, int, struct nameidata *); -int ll_get_acl(struct inode *, struct posix_acl **acl, +int ll_get_acl(struct inode *, struct posix_acl **acl, struct ptlrpc_request **req); int ll_refresh_lsm(struct inode *inode, struct lov_stripe_md *lsm); int ll_extent_lock(struct ll_file_data *, struct inode *, @@ -340,7 +375,7 @@ extern struct super_operations lustre_super_operations; char *ll_read_opt(const char *opt, char *data); int ll_set_opt(const char *opt, char *data, int fl); -void ll_options(char *options, char **ost, char **mds, char **gss, +void ll_options(char *options, char **ost, char **mds, char **gss, char **mds_sec, char **oss_sec, int *async, int *flags); void ll_lli_init(struct ll_inode_info *lli); int ll_fill_super(struct super_block *sb, void *data, int silent); @@ -513,7 +548,7 @@ ll_inode2id(struct lustre_id *id, struct inode *inode) id_fid(lid)); } -static inline void +static inline void ll_prepare_mdc_data(struct mdc_op_data *data, struct inode *i1, struct inode *i2, const char *name, int namelen, int mode) @@ -538,14 +573,14 @@ ll_prepare_mdc_data(struct mdc_op_data *data, struct inode *i1, struct crypto_helper_ops { int (*init_it_key)(struct inode *inode, struct lookup_intent *it); - int (*create_key)(struct inode *dir, mode_t mode, void **key, + int (*create_key)(struct inode *dir, mode_t mode, void **key, int* key_size); - int (*get_mac)(struct inode *inode, struct iattr *iattr, void*value, + int (*get_mac)(struct inode *inode, struct iattr *iattr, void*value, int size, void **key, int* key_size); int (*decrypt_key)(struct inode *inode, struct lookup_intent *it); int (*init_inode_key)(struct inode *inode, void *md_key); int (*destroy_key)(struct inode *inode); -}; +}; /*llite crypto ops for crypto api*/ struct ll_crypto_info { @@ -566,12 +601,12 @@ static inline struct ll_crypto_info* ll_i2crpi(struct inode *inode) static inline struct crypto_helper_ops* ll_i2crpops(struct inode *inode) { - return ll_i2crpi(inode)->ll_cops; + return ll_i2crpi(inode)->ll_cops; } static inline struct crypto_helper_ops* ll_s2crpops(struct super_block *sb) { - return ll_s2crpi(sb)->ll_cops; + return ll_s2crpi(sb)->ll_cops; } static inline struct obd_export *ll_s2gsexp(struct super_block *sb) @@ -586,82 +621,82 @@ static inline struct obd_export *ll_i2gsexp(struct inode *inode) return ll_s2gsexp(inode->i_sb); } -static inline -int ll_crypto_init_it_key(struct inode *inode, struct lookup_intent *it) +static inline +int ll_crypto_init_it_key(struct inode *inode, struct lookup_intent *it) { struct ll_crypto_info *lci = ll_i2crpi(inode); if (lci) { struct crypto_helper_ops *ops = lci->ll_cops;; if (ops && ops->init_it_key) - return ops->init_it_key(inode, it); - } + return ops->init_it_key(inode, it); + } return 0; } -static inline -int ll_crypto_create_key(struct inode *inode, mode_t mode, void **key, - int* key_size) +static inline +int ll_crypto_create_key(struct inode *inode, mode_t mode, void **key, + int* key_size) { struct ll_crypto_info *lci = ll_i2crpi(inode); if (lci) { struct crypto_helper_ops *ops = lci->ll_cops;; if (ops && ops->create_key) - return ops->create_key(inode, mode, key, key_size); - } + return ops->create_key(inode, mode, key, key_size); + } return 0; } -static inline -int ll_crypto_get_mac(struct inode *inode, struct iattr *attr, void *acl, - int acl_size, void **mac, int *mac_size) +static inline +int ll_crypto_get_mac(struct inode *inode, struct iattr *attr, void *acl, + int acl_size, void **mac, int *mac_size) { struct ll_crypto_info *lci = ll_i2crpi(inode); if (lci) { struct crypto_helper_ops *ops = lci->ll_cops; if (ops && ops->get_mac) - return ops->get_mac(inode, attr, acl, acl_size, + return ops->get_mac(inode, attr, acl, acl_size, mac, mac_size); - } + } return 0; } -static inline -int ll_crypto_decrypt_key(struct inode *inode, struct lookup_intent *it) +static inline +int ll_crypto_decrypt_key(struct inode *inode, struct lookup_intent *it) { struct ll_crypto_info *lci = ll_i2crpi(inode); if (lci) { struct crypto_helper_ops *ops = lci->ll_cops; if (ops && ops->decrypt_key) - return ops->decrypt_key(inode, it); + return ops->decrypt_key(inode, it); } - return 0; + return 0; } -static inline -int ll_crypto_init_inode_key(struct inode *inode, void *md_key) +static inline +int ll_crypto_init_inode_key(struct inode *inode, void *md_key) { struct ll_crypto_info *lci = ll_i2crpi(inode); if (lci) { struct crypto_helper_ops *ops = lci->ll_cops; if (ops && ops->init_inode_key) - return ops->init_inode_key(inode, md_key); + return ops->init_inode_key(inode, md_key); } - return 0; + return 0; } -static inline -int ll_crypto_destroy_inode_key(struct inode *inode) +static inline +int ll_crypto_destroy_inode_key(struct inode *inode) { struct ll_crypto_info *lci = ll_i2crpi(inode); if (lci) { struct crypto_helper_ops *ops = lci->ll_cops; if (ops && ops->destroy_key) - return ops->destroy_key(inode); + return ops->destroy_key(inode); } - return 0; + return 0; } -int lustre_init_crypto(struct super_block *sb, char *gkc, +int lustre_init_crypto(struct super_block *sb, char *gkc, struct obd_connect_data *data, int async); int lustre_destroy_crypto(struct super_block *sb); int ll_set_sb_gksinfo(struct super_block *sb, char *type); diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index 12fd279..9794f8a 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -514,7 +514,7 @@ static int osc_destroy(struct obd_export *exp, struct obdo *oa, rc = 0; } else { rc = ptlrpc_queue_wait(request); - + if (rc == -ENOENT) rc = 0; @@ -736,11 +736,11 @@ static obd_count cksum_pages(int nob, obd_count page_count, #define osc_decrypt_page(page, off, count) \ osc_crypt_page(page, off, count, DECRYPT_DATA) -/*Put a global call back var here is Ugly, but put it to client_obd - *also seems not a good idea, WangDi*/ -crypt_cb_t osc_crypt_cb = NULL; - -static int osc_crypt_page(struct page *page, obd_off page_off, obd_off count, +/*Put a global call back var here is Ugly, but put it to client_obd + *also seems not a good idea, WangDi*/ +crypt_cb_t osc_crypt_cb = NULL; + +static int osc_crypt_page(struct page *page, obd_off page_off, obd_off count, int flags) { int rc = 0; @@ -748,9 +748,9 @@ static int osc_crypt_page(struct page *page, obd_off page_off, obd_off count, if (osc_crypt_cb != NULL) rc = osc_crypt_cb(page, page_off, count, flags); - if (rc != 0) + if (rc != 0) CERROR("crypt page error %d \n", rc); - RETURN(rc); + RETURN(rc); } static int osc_brw_prep_request(int cmd, struct obd_import *imp,struct obdo *oa, @@ -821,10 +821,10 @@ static int osc_brw_prep_request(int cmd, struct obd_import *imp,struct obdo *oa, pg->pg, pg->pg->private, pg->pg->index, pg->disk_offset, pg_prev->pg, pg_prev->pg->private, pg_prev->pg->index, pg_prev->disk_offset); - + if (opc == OST_WRITE) { osc_encrypt_page(pg->pg, pg->page_offset, pg->count); - } + } ptlrpc_prep_bulk_page(desc, pg->pg, pg->page_offset & ~PAGE_MASK, pg->count); @@ -895,7 +895,7 @@ static int osc_brw_fini_request(struct ptlrpc_request *req, struct obdo *oa, RETURN(-EPROTO); } LASSERT (req->rq_bulk->bd_nob == requested_nob); - osc_decrypt_page(pga->pg, pga->page_offset, + osc_decrypt_page(pga->pg, pga->page_offset, pga->count); RETURN(check_write_rcs(req, requested_nob, niocount, page_count, pga)); @@ -1267,10 +1267,10 @@ static int brw_interpret_oap(struct ptlrpc_request *request, cli = aa->aa_cli; /* in failout recovery we ignore writeback failure and want * to just tell llite to unlock the page and continue */ - if (request->rq_reqmsg->opc == OST_WRITE && + if (request->rq_reqmsg->opc == OST_WRITE && (cli->cl_import == NULL || cli->cl_import->imp_invalid)) { - CDEBUG(D_INODE, "flipping to rc 0 imp %p inv %d\n", - cli->cl_import, + CDEBUG(D_INODE, "flipping to rc 0 imp %p inv %d\n", + cli->cl_import, cli->cl_import ? cli->cl_import->imp_invalid : -1); rc = 0; } @@ -1464,6 +1464,14 @@ static int osc_send_oap_rpc(struct client_obd *cli, struct lov_oinfo *loi, } if (pos == NULL) break; + /* + * Page submitted for IO has to be locked. Either by + * ->ap_make_ready() or by higher layers. + * + * XXX nikita: this assertion should be adjusted when lustre + * starts using PG_writeback for pages being written out. + */ + LASSERT(PageLocked(oap->oap_page)); /* take the page out of our book-keeping */ list_del_init(&oap->oap_pending_item); @@ -1688,8 +1696,8 @@ static void osc_check_rpcs(struct client_obd *cli) while ((loi = osc_next_loi(cli)) != NULL) { LOI_DEBUG(loi, "%lu in flight\n", rpcs_in_flight(cli)); - - if (rpcs_in_flight(cli) >= cli->cl_max_rpcs_in_flight) + + if (rpcs_in_flight(cli) >= cli->cl_max_rpcs_in_flight) break; /* attempt some read/write balancing by alternating between @@ -1786,7 +1794,7 @@ static int osc_enter_cache(struct client_obd *cli, struct lov_oinfo *loi, /* Make sure that there are write rpcs in flight to wait for. This * is a little silly as this object may not have any pending but * other objects sure might. */ - if (cli->cl_w_in_flight) { + if (cli->cl_w_in_flight) { list_add_tail(&ocw.ocw_entry, &cli->cl_cache_waiters); init_waitqueue_head(&ocw.ocw_waitq); ocw.ocw_oap = oap; @@ -2981,7 +2989,7 @@ static int osc_set_info(struct obd_export *exp, obd_count keylen, if (keylen == strlen("crypto_cb") && memcmp(key, "crypto_cb", keylen) == 0) { LASSERT(vallen == sizeof(crypt_cb_t)); - osc_crypt_cb = (crypt_cb_t)val; + osc_crypt_cb = (crypt_cb_t)val; RETURN(0); } @@ -3078,7 +3086,7 @@ static int osc_disconnect(struct obd_export *exp, unsigned long flags) } static int osc_import_event(struct obd_device *obd, - struct obd_import *imp, + struct obd_import *imp, enum obd_import_event event) { struct client_obd *cli; @@ -3091,7 +3099,7 @@ static int osc_import_event(struct obd_device *obd, /* Only do this on the MDS OSC's */ if (imp->imp_server_timeout) { struct osc_creator *oscc = &obd->u.cli.cl_oscc; - + spin_lock(&oscc->oscc_lock); oscc->oscc_flags |= OSCC_FLAG_RECOVERING; spin_unlock(&oscc->oscc_lock); @@ -3114,7 +3122,7 @@ static int osc_import_event(struct obd_device *obd, /* all pages go to failing rpcs due to the invalid import */ osc_check_rpcs(cli); spin_unlock(&cli->cl_loi_list_lock); - + ldlm_namespace_cleanup(ns, LDLM_FL_LOCAL_ONLY); break;