CFS_MODULE_PARM(oss_num_create_threads, "i", int, 0444,
"number of OSS create threads to start");
+static char *oss_cpts;
+CFS_MODULE_PARM(oss_cpts, "s", charp, 0444,
+ "CPU partitions OSS threads should run on");
+
+static char *oss_io_cpts;
+CFS_MODULE_PARM(oss_io_cpts, "s", charp, 0444,
+ "CPU partitions OSS IO threads should run on");
+
+/*
+ * this page is allocated statically when module is initializing
+ * it is used to simulate data corruptions, see ost_checksum_bulk()
+ * for details. as the original pages provided by the layers below
+ * can be remain in the internal cache, we do not want to modify
+ * them.
+ */
+static struct page *ost_page_to_corrupt = NULL;
+
/**
* Do not return server-side uid/gid to remote client
*/
if (req->rq_status != 0)
CERROR("ost: statfs failed: rc %d\n", req->rq_status);
+ if (OBD_FAIL_CHECK(OBD_FAIL_OST_STATFS_EINPROGRESS))
+ req->rq_status = -EINPROGRESS;
+
RETURN(0);
}
}
static __u32 ost_checksum_bulk(struct ptlrpc_bulk_desc *desc, int opc,
- cksum_type_t cksum_type)
+ cksum_type_t cksum_type)
{
- __u32 cksum;
- int i;
+ struct cfs_crypto_hash_desc *hdesc;
+ unsigned int bufsize;
+ int i, err;
+ unsigned char cfs_alg = cksum_obd2cfs(cksum_type);
+ __u32 cksum;
+
+ hdesc = cfs_crypto_hash_init(cfs_alg, NULL, 0);
+ if (IS_ERR(hdesc)) {
+ CERROR("Unable to initialize checksum hash %s\n",
+ cfs_crypto_hash_name(cfs_alg));
+ return PTR_ERR(hdesc);
+ }
+ CDEBUG(D_INFO, "Checksum for algo %s\n", cfs_crypto_hash_name(cfs_alg));
+ for (i = 0; i < desc->bd_iov_count; i++) {
+
+ /* corrupt the data before we compute the checksum, to
+ * simulate a client->OST data error */
+ if (i == 0 && opc == OST_WRITE &&
+ OBD_FAIL_CHECK(OBD_FAIL_OST_CHECKSUM_RECEIVE)) {
+ int off = desc->bd_iov[i].kiov_offset & ~CFS_PAGE_MASK;
+ int len = desc->bd_iov[i].kiov_len;
+ struct page *np = ost_page_to_corrupt;
+ char *ptr = kmap(desc->bd_iov[i].kiov_page) + off;
+
+ if (np) {
+ char *ptr2 = kmap(np) + off;
+
+ memcpy(ptr2, ptr, len);
+ memcpy(ptr2, "bad3", min(4, len));
+ kunmap(np);
+ desc->bd_iov[i].kiov_page = np;
+ } else {
+ CERROR("can't alloc page for corruption\n");
+ }
+ }
+ cfs_crypto_hash_update_page(hdesc, desc->bd_iov[i].kiov_page,
+ desc->bd_iov[i].kiov_offset & ~CFS_PAGE_MASK,
+ desc->bd_iov[i].kiov_len);
+
+ /* corrupt the data after we compute the checksum, to
+ * simulate an OST->client data error */
+ if (i == 0 && opc == OST_READ &&
+ OBD_FAIL_CHECK(OBD_FAIL_OST_CHECKSUM_SEND)) {
+ int off = desc->bd_iov[i].kiov_offset & ~CFS_PAGE_MASK;
+ int len = desc->bd_iov[i].kiov_len;
+ struct page *np = ost_page_to_corrupt;
+ char *ptr = kmap(desc->bd_iov[i].kiov_page) + off;
+
+ if (np) {
+ char *ptr2 = kmap(np) + off;
+
+ memcpy(ptr2, ptr, len);
+ memcpy(ptr2, "bad4", min(4, len));
+ kunmap(np);
+ desc->bd_iov[i].kiov_page = np;
+ } else {
+ CERROR("can't alloc page for corruption\n");
+ }
+ }
+ }
- cksum = init_checksum(cksum_type);
- for (i = 0; i < desc->bd_iov_count; i++) {
- struct page *page = desc->bd_iov[i].kiov_page;
- int off = desc->bd_iov[i].kiov_offset & ~CFS_PAGE_MASK;
- char *ptr = kmap(page) + off;
- int len = desc->bd_iov[i].kiov_len;
-
- /* corrupt the data before we compute the checksum, to
- * simulate a client->OST data error */
- if (i == 0 && opc == OST_WRITE &&
- OBD_FAIL_CHECK(OBD_FAIL_OST_CHECKSUM_RECEIVE))
- memcpy(ptr, "bad3", min(4, len));
- cksum = compute_checksum(cksum, ptr, len, cksum_type);
- /* corrupt the data after we compute the checksum, to
- * simulate an OST->client data error */
- if (i == 0 && opc == OST_READ &&
- OBD_FAIL_CHECK(OBD_FAIL_OST_CHECKSUM_SEND)) {
- memcpy(ptr, "bad4", min(4, len));
- /* nobody should use corrupted page again */
- ClearPageUptodate(page);
- }
- kunmap(page);
- }
+ bufsize = 4;
+ err = cfs_crypto_hash_final(hdesc, (unsigned char *)&cksum, &bufsize);
+ if (err)
+ cfs_crypto_hash_final(hdesc, NULL, NULL);
- return fini_checksum(cksum, cksum_type);
+ return cksum;
}
static int ost_brw_lock_get(int mode, struct obd_export *exp,
nob += page_rc;
if (page_rc != 0) { /* some data! */
LASSERT (local_nb[i].page != NULL);
- ptlrpc_prep_bulk_page(desc, local_nb[i].page,
- local_nb[i].offset & ~CFS_PAGE_MASK,
- page_rc);
+ ptlrpc_prep_bulk_page_nopin(desc, local_nb[i].page,
+ local_nb[i].lnb_page_offset,
+ page_rc);
}
if (page_rc != local_nb[i].len) { /* short read */
ost_tls_put(req);
out_bulk:
if (desc && !CFS_FAIL_PRECHECK(OBD_FAIL_PTLRPC_CLIENT_BULK_CB2))
- ptlrpc_free_bulk(desc);
+ ptlrpc_free_bulk_nopin(desc);
out:
LASSERT(rc <= 0);
if (rc == 0) {
/* NB Having prepped, we must commit... */
for (i = 0; i < npages; i++)
- ptlrpc_prep_bulk_page(desc, local_nb[i].page,
- local_nb[i].offset & ~CFS_PAGE_MASK,
- local_nb[i].len);
+ ptlrpc_prep_bulk_page_nopin(desc, local_nb[i].page,
+ local_nb[i].lnb_page_offset,
+ local_nb[i].len);
rc = sptlrpc_svc_prep_bulk(req, desc);
if (rc != 0)
body->oa.o_id,
body->oa.o_valid & OBD_MD_FLGROUP ?
body->oa.o_seq : (__u64)0,
- local_nb[0].offset,
- local_nb[npages-1].offset +
+ local_nb[0].lnb_file_offset,
+ local_nb[npages-1].lnb_file_offset +
local_nb[npages-1].len - 1 );
CERROR("client csum %x, original server csum %x, "
"server csum now %x\n",
ost_tls_put(req);
out_bulk:
if (desc)
- ptlrpc_free_bulk(desc);
+ ptlrpc_free_bulk_nopin(desc);
out:
if (rc == 0) {
oti_to_request(oti, req);
RETURN(rc);
}
-#ifdef HAVE_QUOTA_SUPPORT
static int ost_handle_quotactl(struct ptlrpc_request *req)
{
struct obd_quotactl *oqctl, *repoqc;
if (rc)
RETURN(-ENOMEM);
- req->rq_status = obd_quotacheck(req->rq_export, oqctl);
- RETURN(0);
+ /* deprecated, not used any more */
+ req->rq_status = -EOPNOTSUPP;
+ RETURN(-EOPNOTSUPP);
}
-static int ost_handle_quota_adjust_qunit(struct ptlrpc_request *req)
-{
- struct quota_adjust_qunit *oqaq, *repoqa;
- struct lustre_quota_ctxt *qctxt;
- int rc;
- ENTRY;
-
- qctxt = &req->rq_export->exp_obd->u.obt.obt_qctxt;
- oqaq = req_capsule_client_get(&req->rq_pill, &RMF_QUOTA_ADJUST_QUNIT);
- if (oqaq == NULL)
- GOTO(out, rc = -EPROTO);
-
- rc = req_capsule_server_pack(&req->rq_pill);
- if (rc)
- GOTO(out, rc);
-
- repoqa = req_capsule_server_get(&req->rq_pill, &RMF_QUOTA_ADJUST_QUNIT);
- req->rq_status = obd_quota_adjust_qunit(req->rq_export, oqaq, qctxt, NULL);
- *repoqa = *oqaq;
-
- out:
- RETURN(rc);
-}
-#endif
-
static int ost_llog_handle_connect(struct obd_export *exp,
struct ptlrpc_request *req)
{
oa->o_seq = lock->l_resource->lr_name.name[1];
oa->o_valid = OBD_MD_FLID|OBD_MD_FLGROUP;
oinfo->oi_oa = oa;
+ oinfo->oi_capa = BYPASS_CAPA;
rc = obd_sync(&env, lock->l_export, oinfo,
lock->l_policy_data.l_extent.start,
case OST_SYNC:
case OST_SET_INFO:
case OST_GET_INFO:
-#ifdef HAVE_QUOTA_SUPPORT
case OST_QUOTACHECK:
case OST_QUOTACTL:
case OST_QUOTA_ADJUST_QUNIT:
-#endif
rc = lustre_msg_check_version(msg, LUSTRE_OST_VERSION);
if (rc)
CERROR("bad opc %u version %08x, expecting %08x\n",
req_capsule_set(&req->rq_pill, &RQF_OST_GET_INFO_GENERIC);
rc = ost_get_info(req->rq_export, req);
break;
-#ifdef HAVE_QUOTA_SUPPORT
case OST_QUOTACHECK:
CDEBUG(D_INODE, "quotacheck\n");
req_capsule_set(&req->rq_pill, &RQF_OST_QUOTACHECK);
RETURN(0);
rc = ost_handle_quotactl(req);
break;
- case OST_QUOTA_ADJUST_QUNIT:
- CDEBUG(D_INODE, "quota_adjust_qunit\n");
- req_capsule_set(&req->rq_pill, &RQF_OST_QUOTA_ADJUST_QUNIT);
- rc = ost_handle_quota_adjust_qunit(req);
- break;
-#endif
case OBD_PING:
DEBUG_REQ(D_INODE, req, "ping");
req_capsule_set(&req->rq_pill, &RQF_OBD_PING);
LASSERT(thread != NULL);
LASSERT(thread->t_data == NULL);
- LASSERTF(thread->t_id <= OSS_THREADS_MAX, "%u\n", thread->t_id);
OBD_ALLOC_PTR(tls);
if (tls == NULL)
#define OST_WATCHDOG_TIMEOUT (obd_timeout * 1000)
+static struct cfs_cpt_table *ost_io_cptable;
+
/* Sigh - really, this is an OSS, the _server_, not the _target_ */
static int ost_setup(struct obd_device *obd, struct lustre_cfg* lcfg)
{
static struct ptlrpc_service_conf svc_conf;
struct ost_obd *ost = &obd->u.ost;
struct lprocfs_static_vars lvars;
- int oss_min_threads = OSS_THREADS_MIN;
- int oss_max_threads = OSS_THREADS_MAX;
+ nodemask_t *mask;
int rc;
ENTRY;
cfs_mutex_init(&ost->ost_health_mutex);
- if (oss_num_threads == 0) {
- /* Base min threads on memory and cpus */
- oss_min_threads =
- cfs_num_online_cpus() * CFS_NUM_CACHEPAGES >>
- (27 - CFS_PAGE_SHIFT);
- if (oss_min_threads < OSS_THREADS_MIN)
- oss_min_threads = OSS_THREADS_MIN;
- /* Insure a 4x range for dynamic threads */
- if (oss_min_threads > OSS_THREADS_MAX / 4)
- oss_min_threads = OSS_THREADS_MAX / 4;
- oss_max_threads = min(OSS_THREADS_MAX, oss_min_threads * 4 + 1);
- }
-
svc_conf = (typeof(svc_conf)) {
.psc_name = LUSTRE_OSS_NAME,
.psc_watchdog_factor = OSS_SERVICE_WATCHDOG_FACTOR,
},
.psc_thr = {
.tc_thr_name = "ll_ost",
- .tc_nthrs_min = oss_min_threads,
- .tc_nthrs_max = oss_max_threads,
+ .tc_thr_factor = OSS_THR_FACTOR,
+ .tc_nthrs_init = OSS_NTHRS_INIT,
+ .tc_nthrs_base = OSS_NTHRS_BASE,
+ .tc_nthrs_max = OSS_NTHRS_MAX,
.tc_nthrs_user = oss_num_threads,
+ .tc_cpu_affinity = 1,
.tc_ctx_tags = LCT_DT_THREAD,
},
+ .psc_cpt = {
+ .cc_pattern = oss_cpts,
+ },
.psc_ops = {
.so_req_handler = ost_handle,
.so_req_printer = target_print_req,
+ .so_hpreq_handler = ptlrpc_hpreq_handler,
},
};
ost->ost_service = ptlrpc_register_service(&svc_conf,
},
.psc_thr = {
.tc_thr_name = "ll_ost_create",
- .tc_nthrs_min = OSS_CR_THREADS_MIN,
- .tc_nthrs_max = OSS_CR_THREADS_MAX,
+ .tc_thr_factor = OSS_CR_THR_FACTOR,
+ .tc_nthrs_init = OSS_CR_NTHRS_INIT,
+ .tc_nthrs_base = OSS_CR_NTHRS_BASE,
+ .tc_nthrs_max = OSS_CR_NTHRS_MAX,
.tc_nthrs_user = oss_num_create_threads,
+ .tc_cpu_affinity = 1,
.tc_ctx_tags = LCT_DT_THREAD,
},
+ .psc_cpt = {
+ .cc_pattern = oss_cpts,
+ },
.psc_ops = {
.so_req_handler = ost_handle,
.so_req_printer = target_print_req,
+ .so_hpreq_handler = NULL,
},
};
ost->ost_create_service = ptlrpc_register_service(&svc_conf,
GOTO(out_service, rc);
}
+ mask = cfs_cpt_table->ctb_nodemask;
+ /* event CPT feature is disabled in libcfs level by set partition
+ * number to 1, we still want to set node affinity for io service */
+ if (cfs_cpt_number(cfs_cpt_table) == 1 && nodes_weight(*mask) > 1) {
+ int cpt = 0;
+ int i;
+
+ ost_io_cptable = cfs_cpt_table_alloc(nodes_weight(*mask));
+ for_each_node_mask(i, *mask) {
+ if (ost_io_cptable == NULL) {
+ CWARN("OSS failed to create CPT table\n");
+ break;
+ }
+
+ rc = cfs_cpt_set_node(ost_io_cptable, cpt++, i);
+ if (!rc) {
+ CWARN("OSS Failed to set node %d for"
+ "IO CPT table\n", i);
+ cfs_cpt_table_free(ost_io_cptable);
+ ost_io_cptable = NULL;
+ break;
+ }
+ }
+ }
+
memset(&svc_conf, 0, sizeof(svc_conf));
svc_conf = (typeof(svc_conf)) {
.psc_name = "ost_io",
},
.psc_thr = {
.tc_thr_name = "ll_ost_io",
- .tc_nthrs_min = oss_min_threads,
- .tc_nthrs_max = oss_max_threads,
+ .tc_thr_factor = OSS_THR_FACTOR,
+ .tc_nthrs_init = OSS_NTHRS_INIT,
+ .tc_nthrs_base = OSS_NTHRS_BASE,
+ .tc_nthrs_max = OSS_NTHRS_MAX,
.tc_nthrs_user = oss_num_threads,
.tc_cpu_affinity = 1,
.tc_ctx_tags = LCT_DT_THREAD,
},
+ .psc_cpt = {
+ .cc_cptable = ost_io_cptable,
+ .cc_pattern = ost_io_cptable == NULL ?
+ oss_io_cpts : NULL,
+ },
.psc_ops = {
.so_thr_init = ost_thread_init,
.so_thr_done = ost_thread_done,
.so_req_handler = ost_handle,
.so_hpreq_handler = ost_hpreq_handler,
.so_req_printer = target_print_req,
+ .so_hpreq_handler = NULL,
},
};
ost->ost_io_service = ptlrpc_register_service(&svc_conf,
ost->ost_create_service = NULL;
ost->ost_io_service = NULL;
- cfs_mutex_unlock(&ost->ost_health_mutex);
+ cfs_mutex_unlock(&ost->ost_health_mutex);
- lprocfs_obd_cleanup(obd);
+ lprocfs_obd_cleanup(obd);
+
+ if (ost_io_cptable != NULL) {
+ cfs_cpt_table_free(ost_io_cptable);
+ ost_io_cptable = NULL;
+ }
- RETURN(err);
+ RETURN(err);
}
static int ost_health_check(const struct lu_env *env, struct obd_device *obd)
int rc;
ENTRY;
+ ost_page_to_corrupt = cfs_alloc_page(CFS_ALLOC_STD);
+
lprocfs_ost_init_vars(&lvars);
rc = class_register_type(&ost_obd_ops, NULL, lvars.module_vars,
LUSTRE_OSS_NAME, NULL);
static void /*__exit*/ ost_exit(void)
{
+ if (ost_page_to_corrupt)
+ page_cache_release(ost_page_to_corrupt);
+
class_unregister_type(LUSTRE_OSS_NAME);
}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff