From: Alex Zhuravlev Date: Mon, 1 Apr 2024 11:37:32 +0000 (+0300) Subject: LU-17668 ptlrpc: create env in few more threads X-Git-Tag: 2.16.51~31 X-Git-Url: https://git.whamcloud.com/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F42%2F54642%2F32;p=fs%2Flustre-release.git LU-17668 ptlrpc: create env in few more threads like evict thread, pinger, mount process. This avoids cases where there is no lu_env found for a thread: mdt_dom_lvbo_update()) ASSERTION( env ) failed yet another case is when MDT destroys exports - we don't need to re-allocate a new env (including thread infos for every layer) for each export being destroyed. Signed-off-by: Alex Zhuravlev Change-Id: Iff7b757ad34063a3596a34998489cc3c635a02d2 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54642 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin --- diff --git a/lustre/ldlm/ldlm_lockd.c b/lustre/ldlm/ldlm_lockd.c index 3e15d5c..350d3b9 100644 --- a/lustre/ldlm/ldlm_lockd.c +++ b/lustre/ldlm/ldlm_lockd.c @@ -155,7 +155,8 @@ static inline int have_expired_locks(void) static int expired_lock_main(void *arg) { struct list_head *expired = &expired_lock_list; - int do_dump; + struct lu_env env; + int rc, do_dump; ENTRY; @@ -167,7 +168,17 @@ static int expired_lock_main(void *arg) have_expired_locks() || expired_lock_thread_state == ELT_TERMINATE); + rc = lu_env_init(&env, LCT_DT_THREAD | LCT_MD_THREAD); + if (rc) { + CERROR("can't init env: rc=%d\n", rc); + schedule_timeout(HZ * 3); + continue; + } + rc = lu_env_add(&env); + LASSERT(rc == 0); + spin_lock_bh(&waiting_locks_spinlock); + if (expired_lock_dump) { spin_unlock_bh(&waiting_locks_spinlock); @@ -255,6 +266,9 @@ static int expired_lock_main(void *arg) } spin_unlock_bh(&waiting_locks_spinlock); + lu_env_remove(&env); + lu_env_fini(&env); + if (do_dump) { CERROR("dump the log upon eviction\n"); libcfs_debug_dumplog(); diff --git a/lustre/mdt/mdt_coordinator.c b/lustre/mdt/mdt_coordinator.c index 147dcaf..65fce7d 100644 --- a/lustre/mdt/mdt_coordinator.c +++ b/lustre/mdt/mdt_coordinator.c @@ -391,8 +391,7 @@ static int mdt_cdt_started_cb(const struct lu_env *env, larr->arr_status = ARS_CANCELED; larr->arr_req_change = now; - rc = llog_write(hsd->hsd_mti->mti_env, llh, &larr->arr_hdr, - larr->arr_hdr.lrh_index); + rc = llog_write(env, llh, &larr->arr_hdr, larr->arr_hdr.lrh_index); if (rc < 0) { CERROR("%s: cannot update agent log: rc = %d\n", mdt_obd_name(mdt), rc); diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 6c53029..25f27e8 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -7182,7 +7182,7 @@ static int mdt_export_cleanup(struct obd_export *exp) struct obd_device *obd = exp->exp_obd; struct mdt_device *mdt; struct mdt_thread_info *info; - struct lu_env env; + struct lu_env *env; struct mdt_file_data *mfd, *n; int rc = 0; @@ -7204,16 +7204,15 @@ static int mdt_export_cleanup(struct obd_export *exp) mdt = mdt_dev(obd->obd_lu_dev); LASSERT(mdt != NULL); - rc = lu_env_init(&env, LCT_MD_THREAD); - if (rc) - RETURN(rc); + env = lu_env_find(); + LASSERT(env); - info = lu_context_key_get(&env.le_ctx, &mdt_thread_key); + info = lu_context_key_get(&env->le_ctx, &mdt_thread_key); LASSERT(info != NULL); - memset(info, 0, sizeof(*info)); - info->mti_env = &env; + info->mti_env = env; info->mti_mdt = mdt; info->mti_exp = exp; + info->mti_pill = NULL; if (!list_empty(&closing_list)) { struct md_attr *ma = &info->mti_attr; @@ -7238,7 +7237,7 @@ static int mdt_export_cleanup(struct obd_export *exp) * dirty. */ if (mfd->mfd_open_flags & MDS_FMODE_WRITE) - rc = mdt_ctxt_add_dirty_flag(&env, info, mfd); + rc = mdt_ctxt_add_dirty_flag(env, info, mfd); /* Don't unlink orphan on failover umount, LU-184 */ if (exp->exp_flags & OBD_OPT_FAILOVER || @@ -7254,8 +7253,7 @@ static int mdt_export_cleanup(struct obd_export *exp) /* cleanup client slot early */ /* Do not erase record for recoverable client. */ if (!(exp->exp_flags & OBD_OPT_FAILOVER) || exp->exp_failed) - tgt_client_del(&env, exp); - lu_env_fini(&env); + tgt_client_del(env, exp); RETURN(rc); } diff --git a/lustre/mgs/mgs_handler.c b/lustre/mgs/mgs_handler.c index 62632ff..9fbb4f4 100644 --- a/lustre/mgs/mgs_handler.c +++ b/lustre/mgs/mgs_handler.c @@ -872,6 +872,8 @@ static int mgs_iocontrol(unsigned int cmd, struct obd_export *exp, int len, rc = lu_env_init(&env, LCT_MG_THREAD); if (rc) RETURN(rc); + rc = lu_env_add(&env); + LASSERT(rc == 0); rc = -EINVAL; switch (cmd) { @@ -1007,6 +1009,7 @@ out_free: break; } out: + lu_env_remove(&env); lu_env_fini(&env); RETURN(rc); } diff --git a/lustre/mgs/mgs_llog.c b/lustre/mgs/mgs_llog.c index b930532..fe1ba28 100644 --- a/lustre/mgs/mgs_llog.c +++ b/lustre/mgs/mgs_llog.c @@ -1541,8 +1541,8 @@ static int mgs_replace_log(const struct lu_env *env, /* Copy records to this temporary llog */ mrd->temp_llh = orig_llh; - rc = llog_process(env, backup_llh, replace_handler, - (void *)mrd, NULL); + rc = llog_process_or_fork(env, backup_llh, replace_handler, + (void *)mrd, NULL, false); OBD_FREE_PTR(mrd); out_close: rc2 = llog_close(NULL, backup_llh); diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c index f833de7..5ee8fb2 100644 --- a/lustre/obdclass/genops.c +++ b/lustre/obdclass/genops.c @@ -1586,6 +1586,8 @@ int obd_export_evict_by_nid(struct obd_device *obd, const char *nid) struct lnet_nid nid_key; struct obd_export *doomed_exp; int exports_evicted = 0; + struct lu_env *env = NULL, _env; + int rc; libcfs_strnid(&nid_key, nid); @@ -1599,6 +1601,17 @@ int obd_export_evict_by_nid(struct obd_device *obd, const char *nid) } spin_unlock(&obd->obd_dev_lock); + /* can be called via procfs and from ptlrpc */ + env = lu_env_find(); + if (env == NULL) { + rc = lu_env_init(&_env, LCT_DT_THREAD | LCT_MD_THREAD); + if (rc) + return rc; + rc = lu_env_add(&_env); + LASSERT(rc == 0); + env = &_env; + } + doomed_exp = NULL; while (obd_nid_export_for_each(obd, &nid_key, take_first, &doomed_exp) > 0) { @@ -1617,6 +1630,11 @@ int obd_export_evict_by_nid(struct obd_device *obd, const char *nid) doomed_exp = NULL; } + if (env == &_env) { + lu_env_remove(&_env); + lu_env_fini(&_env); + } + if (!exports_evicted) CDEBUG(D_HA, "%s: can't disconnect NID '%s': no exports found\n", @@ -1630,6 +1648,8 @@ int obd_export_evict_by_uuid(struct obd_device *obd, const char *uuid) struct obd_export *doomed_exp = NULL; struct obd_uuid doomed_uuid; int exports_evicted = 0; + struct lu_env env; + int rc; spin_lock(&obd->obd_dev_lock); if (obd->obd_stopping) { @@ -1644,7 +1664,14 @@ int obd_export_evict_by_uuid(struct obd_device *obd, const char *uuid) return exports_evicted; } + rc = lu_env_init(&env, LCT_DT_THREAD | LCT_MD_THREAD); + if (rc) + return rc; + rc = lu_env_add(&env); + LASSERT(rc == 0); + doomed_exp = obd_uuid_lookup(obd, &doomed_uuid); + if (doomed_exp == NULL) { CERROR("%s: can't disconnect %s: no exports found\n", obd->obd_name, uuid); @@ -1657,6 +1684,9 @@ int obd_export_evict_by_uuid(struct obd_device *obd, const char *uuid) exports_evicted++; } + lu_env_remove(&env); + lu_env_fini(&env); + return exports_evicted; } #endif /* HAVE_SERVER_SUPPORT */ diff --git a/lustre/obdclass/llog.c b/lustre/obdclass/llog.c index 24d0132..10d7748 100644 --- a/lustre/obdclass/llog.c +++ b/lustre/obdclass/llog.c @@ -501,23 +501,37 @@ static int llog_process_thread(void *arg) int saved_index = 0; int last_called_index = 0; bool repeated = false; + struct lu_env *env = NULL, _env; ENTRY; if (llh == NULL) RETURN(-EINVAL); - lti = lpi->lpi_env == NULL ? NULL : llog_info(lpi->lpi_env); + /* + * this can be called as a separate thread processing llog or + * as a part of more functional thread like osp sync thread with + * an existing env + */ + env = lu_env_find(); + if (env == NULL) { + rc = lu_env_init(&_env, LCT_DT_THREAD | LCT_MD_THREAD); + if (rc) + RETURN(rc); + rc = lu_env_add(&_env); + if (unlikely(rc)) + RETURN(rc); + env = &_env; + } + lti = llog_info(env); cur_offset = chunk_size = llh->llh_hdr.lrh_len; /* expect chunk_size to be power of two */ LASSERT(is_power_of_2(chunk_size)); OBD_ALLOC_LARGE(buf, chunk_size); - if (buf == NULL) { - lpi->lpi_rc = -ENOMEM; - RETURN(0); - } + if (unlikely(buf == NULL)) + GOTO(out_env, rc = -ENOMEM); last_index = llog_max_idx(llh); if (cd) { @@ -559,7 +573,7 @@ repeat: /* the record index for outdated chunk data */ /* it is safe to process buffer until saved lgh_last_idx */ lh_last_idx = LLOG_HDR_TAIL(llh)->lrt_index; - rc = llog_next_block(lpi->lpi_env, loghandle, &saved_index, + rc = llog_next_block(env, loghandle, &saved_index, index, &cur_offset, buf, chunk_size); if (repeated && rc) CDEBUG(D_OTHER, "cur_offset %llu, chunk_offset %llu," @@ -735,7 +749,7 @@ repeat: } /* using lu_env for passing record offset to * llog_write through various callbacks */ - rc = lpi->lpi_cb(lpi->lpi_env, loghandle, rec, + rc = lpi->lpi_cb(env, loghandle, rec, lpi->lpi_cbdata); last_called_index = index; @@ -748,7 +762,7 @@ repeat: rc == LLOG_SKIP_PLAIN) { GOTO(out, rc); } else if (rc == LLOG_DEL_RECORD) { - rc = llog_cancel_rec(lpi->lpi_env, + rc = llog_cancel_rec(env, loghandle, rec->lrh_index); /* Allow parallel cancelling, ENOENT @@ -807,7 +821,7 @@ out: while (index <= last_index) { if (test_bit_le(index, LLOG_HDR_BITMAP(llh)) != 0) - llog_cancel_rec(lpi->lpi_env, loghandle, + llog_cancel_rec(env, loghandle, index); index++; } @@ -816,6 +830,12 @@ out: } OBD_FREE_LARGE(buf, chunk_size); +out_env: + if (env == &_env) { + lu_env_remove(&_env); + lu_env_fini(&_env); + } + lpi->lpi_rc = rc; return 0; } @@ -823,7 +843,6 @@ out: static int llog_process_thread_daemonize(void *arg) { struct llog_process_info *lpi = arg; - struct lu_env env; int rc; struct nsproxy *new_ns, *curr_ns = current->nsproxy; @@ -844,16 +863,9 @@ static int llog_process_thread_daemonize(void *arg) task_unlock(lpi->lpi_reftask); unshare_fs_struct(); - /* client env has no keys, tags is just 0 */ - rc = lu_env_init(&env, LCT_LOCAL | LCT_MG_THREAD); - if (rc) - goto out; - lpi->lpi_env = &env; rc = llog_process_thread(arg); - lu_env_fini(&env); -out: complete(&lpi->lpi_completion); return rc; } @@ -891,7 +903,6 @@ int llog_process_or_fork(const struct lu_env *env, /* The new thread can't use parent env, * init the new one in llog_process_thread_daemonize. */ - lpi->lpi_env = NULL; init_completion(&lpi->lpi_completion); /* take reference to current, so that * llog_process_thread_daemonize() can use it to switch to @@ -907,7 +918,6 @@ int llog_process_or_fork(const struct lu_env *env, } wait_for_completion(&lpi->lpi_completion); } else { - lpi->lpi_env = env; llog_process_thread(lpi); } rc = lpi->lpi_rc; diff --git a/lustre/obdclass/llog_internal.h b/lustre/obdclass/llog_internal.h index 9ce48ea..d8e2705 100644 --- a/lustre/obdclass/llog_internal.h +++ b/lustre/obdclass/llog_internal.h @@ -23,7 +23,6 @@ struct llog_process_info { void *lpi_catdata; int lpi_rc; struct completion lpi_completion; - const struct lu_env *lpi_env; struct task_struct *lpi_reftask; }; diff --git a/lustre/osp/osp_sync.c b/lustre/osp/osp_sync.c index d56c55c..b65db08 100644 --- a/lustre/osp/osp_sync.c +++ b/lustre/osp/osp_sync.c @@ -1303,7 +1303,6 @@ static int osp_sync_process_queues(const struct lu_env *env, struct osp_sync_args { struct osp_device *osa_dev; - struct lu_env osa_env; struct completion *osa_started; }; @@ -1334,14 +1333,24 @@ static int osp_sync_thread(void *_args) struct llog_ctxt *ctxt; struct obd_device *obd = d->opd_obd; struct llog_handle *llh; - struct lu_env *env = &args->osa_env; struct osp_job_args *ja, *tmp; + struct lu_env env; int rc, count; bool wrapped; ENTRY; complete(args->osa_started); + + rc = lu_env_init(&env, LCT_LOCAL); + if (rc) { + CERROR("%s: can't initialize env: rc = %d\n", + d->opd_obd->obd_name, rc); + RETURN(rc); + } + rc = lu_env_add(&env); + LASSERT(rc == 0); + again: ctxt = llog_get_context(obd, LLOG_MDS_OST_ORIG_CTXT); if (ctxt == NULL) { @@ -1371,7 +1380,7 @@ again: rc = -EINPROGRESS; goto next; } - rc = llog_cat_process(env, llh, osp_sync_process_queues, d, + rc = llog_cat_process(&env, llh, osp_sync_process_queues, d, d->opd_sync_last_catalog_idx, 0); next: @@ -1394,8 +1403,8 @@ next: if (rc == -EINPROGRESS) { /* can't access the llog now - OI scrub is trying to fix * underlying issue. let's wait and try again */ - llog_cat_close(env, llh); - rc = llog_cleanup(env, ctxt); + llog_cat_close(&env, llh); + rc = llog_cleanup(&env, ctxt); if (rc) GOTO(out, rc); schedule_timeout_interruptible(cfs_time_seconds(5)); @@ -1423,7 +1432,7 @@ wait: /* wait till all the requests are completed */ count = 0; while (atomic_read(&d->opd_sync_rpcs_in_progress) > 0) { - osp_sync_process_committed(env, d); + osp_sync_process_committed(&env, d); rc = wait_event_idle_timeout( d->opd_sync_waitq, @@ -1439,8 +1448,8 @@ wait: } - llog_cat_close(env, llh); - rc = llog_cleanup(env, ctxt); + llog_cat_close(&env, llh); + rc = llog_cleanup(&env, ctxt); if (rc) CERROR("can't cleanup llog: %d\n", rc); list_for_each_entry_safe(ja, tmp, &d->opd_sync_error_list, @@ -1457,7 +1466,8 @@ out: atomic_read(&d->opd_sync_rpcs_in_flight), list_empty(&d->opd_sync_committed_there) ? "" : "!"); - lu_env_fini(env); + lu_env_remove(&env); + lu_env_fini(&env); if (xchg(&d->opd_sync_task, NULL) == NULL) /* already being waited for */ @@ -1673,13 +1683,6 @@ int osp_sync_init(const struct lu_env *env, struct osp_device *d) GOTO(err_id, rc); } - rc = lu_env_init(&args->osa_env, LCT_LOCAL); - if (rc) { - CERROR("%s: can't initialize env: rc = %d\n", - d->opd_obd->obd_name, rc); - GOTO(err_llog, rc); - } - /* * Start synchronization thread */ @@ -1689,7 +1692,6 @@ int osp_sync_init(const struct lu_env *env, struct osp_device *d) rc = PTR_ERR(task); CERROR("%s: cannot start sync thread: rc = %d\n", d->opd_obd->obd_name, rc); - lu_env_fini(&args->osa_env); GOTO(err_llog, rc); } d->opd_sync_task = task; diff --git a/lustre/ptlrpc/pinger.c b/lustre/ptlrpc/pinger.c index dfcb17b..27b0904 100644 --- a/lustre/ptlrpc/pinger.c +++ b/lustre/ptlrpc/pinger.c @@ -471,6 +471,8 @@ static int ping_evictor_main(void *arg) struct obd_device *obd; struct obd_export *exp; time64_t expire_time; + struct lu_env env; + int rc; ENTRY; unshare_fs_struct(); @@ -485,6 +487,15 @@ static int ping_evictor_main(void *arg) if ((pet_state == PET_TERMINATE) && list_empty(&pet_list)) break; + rc = lu_env_init(&env, LCT_DT_THREAD | LCT_MD_THREAD); + if (rc) { + CERROR("can't init env: rc=%d\n", rc); + schedule_timeout(HZ * 3); + continue; + } + rc = lu_env_add(&env); + LASSERT(rc == 0); + /* * we only get here if pet_exp != NULL, and the end of this * loop is the only place which sets it NULL again, so lock @@ -538,6 +549,9 @@ static int ping_evictor_main(void *arg) } spin_unlock(&obd->obd_dev_lock); + lu_env_remove(&env); + lu_env_fini(&env); + spin_lock(&pet_lock); list_del_init(&obd->obd_evict_list); spin_unlock(&pet_lock); diff --git a/lustre/target/tgt_mount.c b/lustre/target/tgt_mount.c index 8cd9553..549855a 100644 --- a/lustre/target/tgt_mount.c +++ b/lustre/target/tgt_mount.c @@ -1652,6 +1652,8 @@ static void server_put_super(struct super_block *sb) struct lustre_sb_info *lsi = s2lsi(sb); struct obd_device *obd; char *tmpname, *extraname = NULL; + struct lu_env env; + int rc; int tmpname_sz; int lsiflags = lsi->lsi_flags; bool stop_servers = lsi->lsi_server_started; @@ -1677,6 +1679,14 @@ static void server_put_super(struct super_block *sb) tmpname, rc); } + rc = lu_env_init(&env, LCT_DT_THREAD | LCT_MD_THREAD); + if (rc) { + CERROR("can't init env: rc=%d\n", rc); + GOTO(out, rc); + } + rc = lu_env_add(&env); + LASSERT(rc == 0); + /* Stop the target */ if (!test_bit(LMD_FLG_NOSVC, lsi->lsi_lmd->lmd_flags) && (IS_MDT(lsi) || IS_OST(lsi))) { @@ -1765,6 +1775,10 @@ static void server_put_super(struct super_block *sb) OBD_FREE(extraname, strlen(extraname) + 1); } + lu_env_remove(&env); + lu_env_fini(&env); + +out: LCONSOLE(D_WARNING, "server umount %s complete\n", tmpname); OBD_FREE(tmpname, tmpname_sz); EXIT;