From: Jinshan Xiong Date: Mon, 5 Aug 2013 17:23:03 +0000 (-0700) Subject: LU-3565 mdt: a new param to allocate sequences X-Git-Tag: 2.4.90~8 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=919b358468e13a0ba8719bd4a4c135e11017dd8b LU-3565 mdt: a new param to allocate sequences sequence controller (aka master sequence server) exports FLDB in proc/fs/lustre/seq/ctl-*/fldb. the same entry can be used to modify FLDB using available sequences: lctl set_param seq.ctl*.fldb="[0x0000000280000400-0x0000000280000600):0:mdt" Signed-off-by: Alex Zhuravlev Signed-off-by: Jinshan Xiong Change-Id: I2640811c23cacb00e423fcd408318cac2ac89c6f Reviewed-on: http://review.whamcloud.com/7027 Reviewed-by: Andreas Dilger Tested-by: Hudson Tested-by: Maloo --- diff --git a/lustre/fid/fid_handler.c b/lustre/fid/fid_handler.c index becd6da..caea0dc 100644 --- a/lustre/fid/fid_handler.c +++ b/lustre/fid/fid_handler.c @@ -153,6 +153,41 @@ int seq_server_alloc_super(struct lu_server_seq *seq, RETURN(rc); } +int seq_server_alloc_spec(struct lu_server_seq *seq, + struct lu_seq_range *spec, + const struct lu_env *env) +{ + struct lu_seq_range *space = &seq->lss_space; + int rc = -ENOSPC; + ENTRY; + + /* + * In some cases (like recovery after a disaster) + * we may need to allocate sequences manually + * Notice some sequences can be lost if requested + * range doesn't start at the beginning of current + * free space. Also notice it's not possible now + * to allocate sequences out of natural order. + */ + if (spec->lsr_start >= spec->lsr_end) + RETURN(-EINVAL); + if (spec->lsr_flags != LU_SEQ_RANGE_MDT && + spec->lsr_flags != LU_SEQ_RANGE_OST) + RETURN(-EINVAL); + + mutex_lock(&seq->lss_mutex); + if (spec->lsr_start >= space->lsr_start) { + space->lsr_start = spec->lsr_end; + rc = seq_store_update(env, seq, spec, 1 /* sync */); + + LCONSOLE_INFO("%s: "DRANGE" sequences allocated: rc = %d \n", + seq->lss_name, PRANGE(spec), rc); + } + mutex_unlock(&seq->lss_mutex); + + RETURN(rc); +} + static int __seq_set_init(const struct lu_env *env, struct lu_server_seq *seq) { @@ -365,6 +400,8 @@ LU_KEY_INIT_FINI(seq, struct seq_thread_info); /* context key: seq_thread_key */ LU_CONTEXT_KEY_DEFINE(seq, LCT_MD_THREAD | LCT_DT_THREAD); +extern const struct file_operations seq_fld_proc_seq_fops; + static int seq_server_proc_init(struct lu_server_seq *seq) { #ifdef LPROCFS @@ -388,6 +425,16 @@ static int seq_server_proc_init(struct lu_server_seq *seq) GOTO(out_cleanup, rc); } + if (seq->lss_type == LUSTRE_SEQ_CONTROLLER) { + rc = lprocfs_seq_create(seq->lss_proc_dir, "fldb", 0644, + &seq_fld_proc_seq_fops, seq); + if (rc) { + CERROR("%s: Can't create fldb for sequence manager " + "proc: rc = %d\n", seq->lss_name, rc); + GOTO(out_cleanup, rc); + } + } + RETURN(0); out_cleanup: diff --git a/lustre/fid/fid_internal.h b/lustre/fid/fid_internal.h index e4d3de4..cb4f0b1 100644 --- a/lustre/fid/fid_internal.h +++ b/lustre/fid/fid_internal.h @@ -81,6 +81,10 @@ int seq_store_read(struct lu_server_seq *seq, int seq_store_update(const struct lu_env *env, struct lu_server_seq *seq, struct lu_seq_range *out, int sync); +int seq_server_alloc_spec(struct lu_server_seq *seq, + struct lu_seq_range *spec, + const struct lu_env *env); + int fid_server_mod_init(void); void fid_server_mod_exit(void); diff --git a/lustre/fid/lproc_fid.c b/lustre/fid/lproc_fid.c index 67cf63c..63b4c16 100644 --- a/lustre/fid/lproc_fid.c +++ b/lustre/fid/lproc_fid.c @@ -50,6 +50,7 @@ #include #include #include "fid_internal.h" +#include #ifdef LPROCFS /* @@ -67,6 +68,11 @@ seq_proc_write_common(struct file *file, const char *buffer, LASSERT(range != NULL); + if (count == 5 && strcmp(buffer, "clear") == 0) { + memset(range, 0, sizeof(*range)); + RETURN(0); + } + rc = sscanf(buffer, "[%llx - %llx]\n", (long long unsigned *)&tmp.lsr_start, (long long unsigned *)&tmp.lsr_end); @@ -217,6 +223,279 @@ struct lprocfs_vars seq_server_proc_list[] = { seq_server_proc_read_server, NULL, NULL }, { NULL } }; + +struct fld_seq_param { + struct lu_env fsp_env; + struct dt_it *fsp_it; + struct lu_server_fld *fsp_fld; + struct lu_server_seq *fsp_seq; + unsigned int fsp_stop:1; +}; + +/* + * XXX: below is a copy of the functions in lustre/fld/lproc_fld.c. + * we want to avoid this duplication either by exporting the + * functions or merging fid and fld into a single module. + */ +static void *fldb_seq_start(struct seq_file *p, loff_t *pos) +{ + struct fld_seq_param *param = p->private; + struct lu_server_fld *fld; + struct dt_object *obj; + const struct dt_it_ops *iops; + + if (param == NULL || param->fsp_stop) + return NULL; + + fld = param->fsp_fld; + obj = fld->lsf_obj; + LASSERT(obj != NULL); + iops = &obj->do_index_ops->dio_it; + + iops->load(¶m->fsp_env, param->fsp_it, *pos); + + *pos = be64_to_cpu(*(__u64 *)iops->key(¶m->fsp_env, param->fsp_it)); + return param; +} + +static void fldb_seq_stop(struct seq_file *p, void *v) +{ + struct fld_seq_param *param = p->private; + const struct dt_it_ops *iops; + struct lu_server_fld *fld; + struct dt_object *obj; + + if (param == NULL) + return; + + fld = param->fsp_fld; + obj = fld->lsf_obj; + LASSERT(obj != NULL); + iops = &obj->do_index_ops->dio_it; + + iops->put(¶m->fsp_env, param->fsp_it); +} + +static void *fldb_seq_next(struct seq_file *p, void *v, loff_t *pos) +{ + struct fld_seq_param *param = p->private; + struct lu_server_fld *fld; + struct dt_object *obj; + const struct dt_it_ops *iops; + int rc; + + if (param == NULL || param->fsp_stop) + return NULL; + + fld = param->fsp_fld; + obj = fld->lsf_obj; + LASSERT(obj != NULL); + iops = &obj->do_index_ops->dio_it; + + rc = iops->next(¶m->fsp_env, param->fsp_it); + if (rc > 0) { + param->fsp_stop = 1; + return NULL; + } + + *pos = be64_to_cpu(*(__u64 *)iops->key(¶m->fsp_env, param->fsp_it)); + return param; +} + +static int fldb_seq_show(struct seq_file *p, void *v) +{ + struct fld_seq_param *param = p->private; + struct lu_server_fld *fld; + struct dt_object *obj; + const struct dt_it_ops *iops; + struct lu_seq_range fld_rec; + int rc; + + if (param == NULL || param->fsp_stop) + return 0; + + fld = param->fsp_fld; + obj = fld->lsf_obj; + LASSERT(obj != NULL); + iops = &obj->do_index_ops->dio_it; + + rc = iops->rec(¶m->fsp_env, param->fsp_it, + (struct dt_rec *)&fld_rec, 0); + if (rc != 0) { + CERROR("%s: read record error: rc = %d\n", + fld->lsf_name, rc); + } else if (fld_rec.lsr_start != 0) { + range_be_to_cpu(&fld_rec, &fld_rec); + rc = seq_printf(p, DRANGE"\n", PRANGE(&fld_rec)); + } + + return rc; +} + +struct seq_operations fldb_sops = { + .start = fldb_seq_start, + .stop = fldb_seq_stop, + .next = fldb_seq_next, + .show = fldb_seq_show, +}; + +static int fldb_seq_open(struct inode *inode, struct file *file) +{ + struct proc_dir_entry *dp = PDE(inode); + struct seq_file *seq; + struct lu_server_seq *ss = (struct lu_server_seq *)dp->data; + struct lu_server_fld *fld; + struct dt_object *obj; + const struct dt_it_ops *iops; + struct fld_seq_param *param = NULL; + int env_init = 0; + int rc; + + fld = ss->lss_site->ss_server_fld; + LASSERT(fld != NULL); + + LPROCFS_ENTRY_AND_CHECK(dp); + rc = seq_open(file, &fldb_sops); + if (rc) + GOTO(out, rc); + + obj = fld->lsf_obj; + if (obj == NULL) { + seq = file->private_data; + seq->private = NULL; + return 0; + } + + OBD_ALLOC_PTR(param); + if (param == NULL) + GOTO(out, rc = -ENOMEM); + + rc = lu_env_init(¶m->fsp_env, LCT_MD_THREAD); + if (rc != 0) + GOTO(out, rc); + + env_init = 1; + iops = &obj->do_index_ops->dio_it; + param->fsp_it = iops->init(¶m->fsp_env, obj, 0, NULL); + if (IS_ERR(param->fsp_it)) + GOTO(out, rc = PTR_ERR(param->fsp_it)); + + param->fsp_fld = fld; + param->fsp_seq = ss; + param->fsp_stop = 0; + + seq = file->private_data; + seq->private = param; +out: + if (rc != 0) { + if (env_init == 1) + lu_env_fini(¶m->fsp_env); + if (param != NULL) + OBD_FREE_PTR(param); + LPROCFS_EXIT(); + } + return rc; +} + +static int fldb_seq_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct fld_seq_param *param; + struct lu_server_fld *fld; + struct dt_object *obj; + const struct dt_it_ops *iops; + + param = seq->private; + if (param == NULL) { + lprocfs_seq_release(inode, file); + return 0; + } + + fld = param->fsp_fld; + obj = fld->lsf_obj; + LASSERT(obj != NULL); + iops = &obj->do_index_ops->dio_it; + + LASSERT(iops != NULL); + LASSERT(obj != NULL); + LASSERT(param->fsp_it != NULL); + iops->fini(¶m->fsp_env, param->fsp_it); + lu_env_fini(¶m->fsp_env); + OBD_FREE_PTR(param); + lprocfs_seq_release(inode, file); + + return 0; +} + +static ssize_t fldb_seq_write(struct file *file, const char *buf, + size_t len, loff_t *off) +{ + struct seq_file *seq = file->private_data; + struct fld_seq_param *param; + struct lu_seq_range range; + int rc = 0; + char *buffer, *_buffer; + ENTRY; + + param = seq->private; + if (param == NULL) + RETURN(-EINVAL); + + OBD_ALLOC(buffer, len + 1); + if (buffer == NULL) + RETURN(-ENOMEM); + memcpy(buffer, buf, len); + buffer[len] = 0; + _buffer = buffer; + + /* + * format - [0x0000000200000007-0x0000000200000008):0:mdt + */ + if (*buffer != '[') + GOTO(out, rc = -EINVAL); + buffer++; + + range.lsr_start = simple_strtoull(buffer, &buffer, 0); + if (*buffer != '-') + GOTO(out, rc = -EINVAL); + buffer++; + + range.lsr_end = simple_strtoull(buffer, &buffer, 0); + if (*buffer != ')') + GOTO(out, rc = -EINVAL); + buffer++; + if (*buffer != ':') + GOTO(out, rc = -EINVAL); + buffer++; + + range.lsr_index = simple_strtoul(buffer, &buffer, 0); + if (*buffer != ':') + GOTO(out, rc = -EINVAL); + buffer++; + + if (strncmp(buffer, "mdt", 3) == 0) + range.lsr_flags = LU_SEQ_RANGE_MDT; + else if (strncmp(buffer, "ost", 3) == 0) + range.lsr_flags = LU_SEQ_RANGE_OST; + else + GOTO(out, rc = -EINVAL); + + rc = seq_server_alloc_spec(param->fsp_seq->lss_site->ss_control_seq, + &range, ¶m->fsp_env); + +out: + OBD_FREE(_buffer, len + 1); + RETURN(rc < 0 ? rc : len); +} + +const struct file_operations seq_fld_proc_seq_fops = { + .owner = THIS_MODULE, + .open = fldb_seq_open, + .read = seq_read, + .write = fldb_seq_write, + .release = fldb_seq_release, +}; + #endif /* HAVE_SERVER_SUPPORT */ /* Client side procfs stuff */ diff --git a/lustre/fld/lproc_fld.c b/lustre/fld/lproc_fld.c index 97bd3d5..1a3e58e 100644 --- a/lustre/fld/lproc_fld.c +++ b/lustre/fld/lproc_fld.c @@ -55,6 +55,7 @@ #include #include #include "fld_internal.h" +#include #ifdef LPROCFS static int diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index e96cc8f..0a7b06f 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -3816,6 +3816,52 @@ test_67() { #LU-2950 } run_test 67 "test routes conversion and configuration" +test_68() { + local fid + local seq + local START + local END + + [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.4.53) ] || + { skip "Need MDS version at least 2.4.53"; return 0; } + + umount_client $MOUNT || error "umount client failed" + + start_mds || error "MDT start failed" + start_ost + + # START-END - the sequences we'll be reserving + START=$(do_facet $SINGLEMDS \ + lctl get_param -n seq.ctl*.space | awk -F'[[ ]' '{print $2}') + END=$((START + (1 << 30))) + do_facet $SINGLEMDS \ + lctl set_param seq.ctl*.fldb="[$START-$END\):0:mdt" + + # reset the sequences MDT0000 has already assigned + do_facet $SINGLEMDS \ + lctl set_param seq.srv*MDT0000.space=clear + + # remount to let the client allocate new sequence + mount_client $MOUNT || error "mount client failed" + + touch $DIR/$tfile + do_facet $SINGLEMDS \ + lctl get_param seq.srv*MDT0000.space + $LFS path2fid $DIR/$tfile + + local old_ifs="$IFS" + IFS='[:]' + fid=($($LFS path2fid $DIR/$tfile)) + IFS="$old_ifs" + let seq=${fid[1]} + + if [[ $seq < $END ]]; then + error "used reserved sequence $seq?" + fi + cleanup || return $? +} +run_test 68 "be able to reserve specific sequences in FLDB" + test_70a() { [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return local MDTIDX=1