From: wangdi Date: Tue, 15 Oct 2013 04:40:21 +0000 (-0700) Subject: LU-1445 ofd: set index during server_data_init X-Git-Tag: 2.3.59~25 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=3269ac0f5186c3c319d10e8e642ea56e5dbbbdf2 LU-1445 ofd: set index during server_data_init Get mdt/ost index from obd_name, and set it in server_data_init. Signed-off-by: wang di Change-Id: I62466f3bcadb738b4182db90773e83361d6963e9 Reviewed-on: http://review.whamcloud.com/4918 Reviewed-by: Andreas Dilger Tested-by: Hudson Reviewed-by: Mike Pershin Reviewed-by: Alex Zhuravlev Tested-by: Maloo --- diff --git a/lustre/include/lustre_disk.h b/lustre/include/lustre_disk.h index 7869803..db0fdf9 100644 --- a/lustre/include/lustre_disk.h +++ b/lustre/include/lustre_disk.h @@ -327,8 +327,8 @@ struct lr_server_data { __u64 lsd_catalog_oid; /* recovery catalog object id */ __u32 lsd_catalog_ogen; /* recovery catalog inode generation */ __u8 lsd_peeruuid[40]; /* UUID of MDS associated with this OST */ - __u32 lsd_ost_index; /* index number of OST in LOV */ - __u32 lsd_mdt_index; /* index number of MDT in LMV */ + __u32 lsd_osd_index; /* index number of OST in LOV */ + __u32 lsd_padding1; /* was lsd_mdt_index, unused in 2.4.0 */ __u32 lsd_start_epoch; /* VBR: start epoch from last boot */ /** transaction values since lsd_trans_table_time */ __u64 lsd_trans_table[LR_EXPIRE_INTERVALS]; @@ -376,55 +376,55 @@ static inline void check_lcd(char *obd_name, int index, static inline void lsd_le_to_cpu(struct lr_server_data *buf, struct lr_server_data *lsd) { - int i; - memcpy(lsd->lsd_uuid, buf->lsd_uuid, sizeof (lsd->lsd_uuid)); - lsd->lsd_last_transno = le64_to_cpu(buf->lsd_last_transno); - lsd->lsd_compat14 = le64_to_cpu(buf->lsd_compat14); - lsd->lsd_mount_count = le64_to_cpu(buf->lsd_mount_count); - lsd->lsd_feature_compat = le32_to_cpu(buf->lsd_feature_compat); - lsd->lsd_feature_rocompat = le32_to_cpu(buf->lsd_feature_rocompat); - lsd->lsd_feature_incompat = le32_to_cpu(buf->lsd_feature_incompat); - lsd->lsd_server_size = le32_to_cpu(buf->lsd_server_size); - lsd->lsd_client_start = le32_to_cpu(buf->lsd_client_start); - lsd->lsd_client_size = le16_to_cpu(buf->lsd_client_size); - lsd->lsd_subdir_count = le16_to_cpu(buf->lsd_subdir_count); - lsd->lsd_catalog_oid = le64_to_cpu(buf->lsd_catalog_oid); - lsd->lsd_catalog_ogen = le32_to_cpu(buf->lsd_catalog_ogen); - memcpy(lsd->lsd_peeruuid, buf->lsd_peeruuid, sizeof(lsd->lsd_peeruuid)); - lsd->lsd_ost_index = le32_to_cpu(buf->lsd_ost_index); - lsd->lsd_mdt_index = le32_to_cpu(buf->lsd_mdt_index); - lsd->lsd_start_epoch = le32_to_cpu(buf->lsd_start_epoch); - for (i = 0; i < LR_EXPIRE_INTERVALS; i++) - lsd->lsd_trans_table[i] = le64_to_cpu(buf->lsd_trans_table[i]); - lsd->lsd_trans_table_time = le32_to_cpu(buf->lsd_trans_table_time); - lsd->lsd_expire_intervals = le32_to_cpu(buf->lsd_expire_intervals); + int i; + memcpy(lsd->lsd_uuid, buf->lsd_uuid, sizeof(lsd->lsd_uuid)); + lsd->lsd_last_transno = le64_to_cpu(buf->lsd_last_transno); + lsd->lsd_compat14 = le64_to_cpu(buf->lsd_compat14); + lsd->lsd_mount_count = le64_to_cpu(buf->lsd_mount_count); + lsd->lsd_feature_compat = le32_to_cpu(buf->lsd_feature_compat); + lsd->lsd_feature_rocompat = le32_to_cpu(buf->lsd_feature_rocompat); + lsd->lsd_feature_incompat = le32_to_cpu(buf->lsd_feature_incompat); + lsd->lsd_server_size = le32_to_cpu(buf->lsd_server_size); + lsd->lsd_client_start = le32_to_cpu(buf->lsd_client_start); + lsd->lsd_client_size = le16_to_cpu(buf->lsd_client_size); + lsd->lsd_subdir_count = le16_to_cpu(buf->lsd_subdir_count); + lsd->lsd_catalog_oid = le64_to_cpu(buf->lsd_catalog_oid); + lsd->lsd_catalog_ogen = le32_to_cpu(buf->lsd_catalog_ogen); + memcpy(lsd->lsd_peeruuid, buf->lsd_peeruuid, sizeof(lsd->lsd_peeruuid)); + lsd->lsd_osd_index = le32_to_cpu(buf->lsd_osd_index); + lsd->lsd_padding1 = le32_to_cpu(buf->lsd_padding1); + lsd->lsd_start_epoch = le32_to_cpu(buf->lsd_start_epoch); + for (i = 0; i < LR_EXPIRE_INTERVALS; i++) + lsd->lsd_trans_table[i] = le64_to_cpu(buf->lsd_trans_table[i]); + lsd->lsd_trans_table_time = le32_to_cpu(buf->lsd_trans_table_time); + lsd->lsd_expire_intervals = le32_to_cpu(buf->lsd_expire_intervals); } static inline void lsd_cpu_to_le(struct lr_server_data *lsd, struct lr_server_data *buf) { - int i; - memcpy(buf->lsd_uuid, lsd->lsd_uuid, sizeof (buf->lsd_uuid)); - buf->lsd_last_transno = cpu_to_le64(lsd->lsd_last_transno); - buf->lsd_compat14 = cpu_to_le64(lsd->lsd_compat14); - buf->lsd_mount_count = cpu_to_le64(lsd->lsd_mount_count); - buf->lsd_feature_compat = cpu_to_le32(lsd->lsd_feature_compat); - buf->lsd_feature_rocompat = cpu_to_le32(lsd->lsd_feature_rocompat); - buf->lsd_feature_incompat = cpu_to_le32(lsd->lsd_feature_incompat); - buf->lsd_server_size = cpu_to_le32(lsd->lsd_server_size); - buf->lsd_client_start = cpu_to_le32(lsd->lsd_client_start); - buf->lsd_client_size = cpu_to_le16(lsd->lsd_client_size); - buf->lsd_subdir_count = cpu_to_le16(lsd->lsd_subdir_count); - buf->lsd_catalog_oid = cpu_to_le64(lsd->lsd_catalog_oid); - buf->lsd_catalog_ogen = cpu_to_le32(lsd->lsd_catalog_ogen); - memcpy(buf->lsd_peeruuid, lsd->lsd_peeruuid, sizeof(buf->lsd_peeruuid)); - buf->lsd_ost_index = cpu_to_le32(lsd->lsd_ost_index); - buf->lsd_mdt_index = cpu_to_le32(lsd->lsd_mdt_index); - buf->lsd_start_epoch = cpu_to_le32(lsd->lsd_start_epoch); - for (i = 0; i < LR_EXPIRE_INTERVALS; i++) - buf->lsd_trans_table[i] = cpu_to_le64(lsd->lsd_trans_table[i]); - buf->lsd_trans_table_time = cpu_to_le32(lsd->lsd_trans_table_time); - buf->lsd_expire_intervals = cpu_to_le32(lsd->lsd_expire_intervals); + int i; + memcpy(buf->lsd_uuid, lsd->lsd_uuid, sizeof(buf->lsd_uuid)); + buf->lsd_last_transno = cpu_to_le64(lsd->lsd_last_transno); + buf->lsd_compat14 = cpu_to_le64(lsd->lsd_compat14); + buf->lsd_mount_count = cpu_to_le64(lsd->lsd_mount_count); + buf->lsd_feature_compat = cpu_to_le32(lsd->lsd_feature_compat); + buf->lsd_feature_rocompat = cpu_to_le32(lsd->lsd_feature_rocompat); + buf->lsd_feature_incompat = cpu_to_le32(lsd->lsd_feature_incompat); + buf->lsd_server_size = cpu_to_le32(lsd->lsd_server_size); + buf->lsd_client_start = cpu_to_le32(lsd->lsd_client_start); + buf->lsd_client_size = cpu_to_le16(lsd->lsd_client_size); + buf->lsd_subdir_count = cpu_to_le16(lsd->lsd_subdir_count); + buf->lsd_catalog_oid = cpu_to_le64(lsd->lsd_catalog_oid); + buf->lsd_catalog_ogen = cpu_to_le32(lsd->lsd_catalog_ogen); + memcpy(buf->lsd_peeruuid, lsd->lsd_peeruuid, sizeof(buf->lsd_peeruuid)); + buf->lsd_osd_index = cpu_to_le32(lsd->lsd_osd_index); + buf->lsd_padding1 = cpu_to_le32(lsd->lsd_padding1); + buf->lsd_start_epoch = cpu_to_le32(lsd->lsd_start_epoch); + for (i = 0; i < LR_EXPIRE_INTERVALS; i++) + buf->lsd_trans_table[i] = cpu_to_le64(lsd->lsd_trans_table[i]); + buf->lsd_trans_table_time = cpu_to_le32(lsd->lsd_trans_table_time); + buf->lsd_expire_intervals = cpu_to_le32(lsd->lsd_expire_intervals); } static inline void lcd_le_to_cpu(struct lsd_client_data *buf, diff --git a/lustre/mdt/mdt_recovery.c b/lustre/mdt/mdt_recovery.c index 2bccbf6..cfefa30 100644 --- a/lustre/mdt/mdt_recovery.c +++ b/lustre/mdt/mdt_recovery.c @@ -183,6 +183,7 @@ static int mdt_server_data_init(const struct lu_env *env, struct dt_object *obj; struct lu_attr *la; unsigned long last_rcvd_size; + __u32 index; __u64 mount_count; int rc; ENTRY; @@ -193,6 +194,13 @@ static int mdt_server_data_init(const struct lu_env *env, CLASSERT(offsetof(struct lsd_client_data, lcd_padding) + sizeof(lcd->lcd_padding) == LR_CLIENT_SIZE); + rc = server_name2index(obd->obd_name, &index, NULL); + if (rc < 0) { + CERROR("%s: Can not get index from obd_name: rc = %d\n", + obd->obd_name, rc); + RETURN(rc); + } + mti = lu_context_key_get(&env->le_ctx, &mdt_thread_key); LASSERT(mti != NULL); la = &mti->mti_attr.ma_attr; @@ -219,8 +227,9 @@ static int mdt_server_data_init(const struct lu_env *env, lsd->lsd_feature_incompat = OBD_INCOMPAT_MDT | OBD_INCOMPAT_COMMON_LR | OBD_INCOMPAT_MULTI_OI; - } else { - LCONSOLE_WARN("%s: used disk, loading\n", obd->obd_name); + lsd->lsd_osd_index = index; + } else { + LCONSOLE_WARN("%s: used disk, loading\n", obd->obd_name); rc = tgt_server_data_read(env, &mdt->mdt_lut); if (rc) { CERROR("error reading MDS %s: rc %d\n", LAST_RCVD, rc); @@ -236,7 +245,15 @@ static int mdt_server_data_init(const struct lu_env *env, lsd->lsd_feature_compat |= OBD_COMPAT_MDT; lsd->lsd_feature_incompat |= OBD_INCOMPAT_MDT | OBD_INCOMPAT_COMMON_LR; - } + if (lsd->lsd_osd_index != index) { + LCONSOLE_ERROR_MSG(0x157, "%s: index %d in last rcvd is" + "different with the index %d in" + "config log, It might be disk" + "corruption!\n", obd->obd_name, + lsd->lsd_osd_index, index); + GOTO(out, rc = -EINVAL); + } + } mount_count = lsd->lsd_mount_count; if (lsd->lsd_feature_incompat & ~MDT_INCOMPAT_SUPP) { diff --git a/lustre/ofd/ofd_fs.c b/lustre/ofd/ofd_fs.c index cdfd40f..e85ba4d 100644 --- a/lustre/ofd/ofd_fs.c +++ b/lustre/ofd/ofd_fs.c @@ -393,8 +393,9 @@ int ofd_server_data_init(const struct lu_env *env, struct ofd_device *ofd) struct ofd_thread_info *info = ofd_info(env); struct lr_server_data *lsd = &ofd->ofd_lut.lut_lsd; struct obd_device *obd = ofd_obd(ofd); - unsigned long last_rcvd_size; - int rc; + unsigned long last_rcvd_size; + __u32 index; + int rc; rc = dt_attr_get(env, ofd->ofd_lut.lut_last_rcvd, &info->fti_attr, BYPASS_CAPA); @@ -407,6 +408,13 @@ int ofd_server_data_init(const struct lu_env *env, struct ofd_device *ofd) CLASSERT (offsetof(struct lr_server_data, lsd_padding) + sizeof(lsd->lsd_padding) == LR_SERVER_SIZE); + rc = server_name2index(obd->obd_name, &index, NULL); + if (rc < 0) { + CERROR("%s: Can not get index from obd_name: rc = %d\n", + obd->obd_name, rc); + RETURN(rc); + } + if (last_rcvd_size == 0) { LCONSOLE_WARN("%s: new disk, initializing\n", obd->obd_name); @@ -419,6 +427,7 @@ int ofd_server_data_init(const struct lu_env *env, struct ofd_device *ofd) lsd->lsd_client_size = LR_CLIENT_SIZE; lsd->lsd_subdir_count = FILTER_SUBDIR_COUNT; lsd->lsd_feature_incompat = OBD_INCOMPAT_OST; + lsd->lsd_osd_index = index; } else { rc = tgt_server_data_read(env, &ofd->ofd_lut); if (rc) { @@ -434,6 +443,17 @@ int ofd_server_data_init(const struct lu_env *env, struct ofd_device *ofd) obd->obd_uuid.uuid, lsd->lsd_uuid); GOTO(err_fsd, rc = -EINVAL); } + + if (lsd->lsd_osd_index == 0) { + lsd->lsd_osd_index = index; + } else if (lsd->lsd_osd_index != index) { + LCONSOLE_ERROR("%s: index %d in last rcvd is different" + " with the index %d in config log." + " It might be disk corruption!\n", + obd->obd_name, lsd->lsd_osd_index, + index); + GOTO(err_fsd, rc = -EINVAL); + } } lsd->lsd_mount_count++; diff --git a/lustre/ofd/ofd_obd.c b/lustre/ofd/ofd_obd.c index 82f5492..3b16bc9 100644 --- a/lustre/ofd/ofd_obd.c +++ b/lustre/ofd/ofd_obd.c @@ -169,16 +169,9 @@ static int ofd_parse_connect_data(const struct lu_env *env, if (data->ocd_connect_flags & OBD_CONNECT_INDEX) { struct lr_server_data *lsd = &ofd->ofd_lut.lut_lsd; - int index = lsd->lsd_ost_index; + int index = lsd->lsd_osd_index; - if (!(lsd->lsd_feature_compat & OBD_COMPAT_OST)) { - /* this will only happen on the first connect */ - lsd->lsd_ost_index = data->ocd_index; - lsd->lsd_feature_compat |= OBD_COMPAT_OST; - /* sync is not needed here as lut_client_add will - * set exp_need_sync flag */ - tgt_server_data_update(env, &ofd->ofd_lut, 0); - } else if (index != data->ocd_index) { + if (index != data->ocd_index) { LCONSOLE_ERROR_MSG(0x136, "Connection from %s to index" " %u doesn't match actual OST index" " %u in last_rcvd file, bad " @@ -187,8 +180,14 @@ static int ofd_parse_connect_data(const struct lu_env *env, data->ocd_index); RETURN(-EBADF); } + if (!(lsd->lsd_feature_compat & OBD_COMPAT_OST)) { + /* this will only happen on the first connect */ + lsd->lsd_feature_compat |= OBD_COMPAT_OST; + /* sync is not needed here as lut_client_add will + * set exp_need_sync flag */ + tgt_server_data_update(env, &ofd->ofd_lut, 0); + } } - if (OBD_FAIL_CHECK(OBD_FAIL_OST_BRW_SIZE)) { data->ocd_brw_size = 65536; } else if (data->ocd_connect_flags & OBD_CONNECT_BRW_SIZE) { @@ -746,7 +745,7 @@ static int ofd_statfs(const struct lu_env *env, struct obd_export *exp, osfs->os_files, osfs->os_ffree, osfs->os_state); if (OBD_FAIL_CHECK_VALUE(OBD_FAIL_OST_ENOINO, - ofd->ofd_lut.lut_lsd.lsd_ost_index)) + ofd->ofd_lut.lut_lsd.lsd_osd_index)) osfs->os_ffree = 0; /* OS_STATE_READONLY can be set by OSD already */ @@ -766,7 +765,7 @@ static int ofd_statfs(const struct lu_env *env, struct obd_export *exp, } if (OBD_FAIL_CHECK_VALUE(OBD_FAIL_OST_ENOSPC, - ofd->ofd_lut.lut_lsd.lsd_ost_index)) + ofd->ofd_lut.lut_lsd.lsd_osd_index)) osfs->os_bfree = osfs->os_bavail = 2; EXIT; diff --git a/lustre/utils/lr_reader.c b/lustre/utils/lr_reader.c index afaaf33..84346e1 100644 --- a/lustre/utils/lr_reader.c +++ b/lustre/utils/lr_reader.c @@ -153,75 +153,72 @@ int main(int argc, char *const argv[]) goto out_close; } - #if 0 - __u8 lsd_uuid[40]; /* server UUID */ - __u64 lsd_last_transno; /* last completed transaction ID */ - __u64 lsd_compat14; /* reserved - compat with old last_rcvd */ - __u64 lsd_mount_count; /* incarnation number */ - __u32 lsd_feature_compat; /* compatible feature flags */ - __u32 lsd_feature_rocompat;/* read-only compatible feature flags */ - __u32 lsd_feature_incompat;/* incompatible feature flags */ - __u32 lsd_server_size; /* size of server data area */ - __u32 lsd_client_start; /* start of per-client data area */ - __u16 lsd_client_size; /* size of per-client data area */ - __u16 lsd_subdir_count; /* number of subdirectories for objects */ - __u64 lsd_catalog_oid; /* recovery catalog object id */ - __u32 lsd_catalog_ogen; /* recovery catalog inode generation */ - __u8 lsd_peeruuid[40]; /* UUID of MDS associated with this OST */ - __u32 lsd_ost_index; /* index number of OST in LOV */ - __u32 lsd_mdt_index; /* index number of MDT in LMV */ - __u8 lsd_padding[LR_SERVER_SIZE - 148]; - #endif - - printf("UUID %s\n", lsd.lsd_uuid); - printf("Feature compat=%#x\n", lsd.lsd_feature_compat); - printf("Feature incompat=%#x\n", lsd.lsd_feature_incompat); - printf("Feature rocompat=%#x\n", lsd.lsd_feature_rocompat); - printf("Last transaction %llu\n", (long long)lsd.lsd_last_transno); - printf("ost index %u\n", lsd.lsd_ost_index); - printf("mdt index %u\n", lsd.lsd_mdt_index); - - if ((lsd.lsd_feature_compat & OBD_COMPAT_OST) || - (lsd.lsd_feature_incompat & OBD_INCOMPAT_OST)) { - printf("OST, index %d\n", lsd.lsd_ost_index); - } else if ((lsd.lsd_feature_compat & OBD_COMPAT_MDT) || - (lsd.lsd_feature_incompat & OBD_INCOMPAT_MDT)) { - /* We must co-locate so mgs can see old logs. - If user doesn't want this, they can copy the old - logs manually and re-tunefs. */ - printf("MDS, index %d\n", lsd.lsd_mdt_index); - } else { - /* If neither is set, we're pre-1.4.6, make a guess. */ - /* Construct debugfs command line. */ - memset(cmd, 0, sizeof(cmd)); - sprintf(cmd, - "%s -c -R 'rdump /%s %s' %s", - DEBUGFS, MDT_LOGS_DIR, tmpdir, dev); - - run_command(cmd); - - sprintf(filepnm, "%s/%s", tmpdir, MDT_LOGS_DIR); - if (lsd.lsd_ost_index > 0) { - printf("non-flagged OST, index %d\n", - lsd.lsd_ost_index); - } else { - /* If there's a LOGS dir, it's an MDT */ - if ((ret = access(filepnm, F_OK)) == 0) { - /* Old MDT's are always index 0 - (pre CMD) */ - printf("non-flagged MDS, index 0\n"); - } else { - printf("non-flagged OST, index unknown\n"); - } - } - } - +#if 0 + __u8 lsd_uuid[40]; /* server UUID */ + __u64 lsd_last_transno; /* last completed transaction ID */ + __u64 lsd_compat14; /* reserved - compat with old last_rcvd */ + __u64 lsd_mount_count; /* incarnation number */ + __u32 lsd_feature_compat; /* compatible feature flags */ + __u32 lsd_feature_rocompat;/* read-only compatible feature flags */ + __u32 lsd_feature_incompat;/* incompatible feature flags */ + __u32 lsd_server_size; /* size of server data area */ + __u32 lsd_client_start; /* start of per-client data area */ + __u16 lsd_client_size; /* size of per-client data area */ + __u16 lsd_subdir_count; /* number of subdirectories for objects */ + __u64 lsd_catalog_oid; /* recovery catalog object id */ + __u32 lsd_catalog_ogen; /* recovery catalog inode generation */ + __u8 lsd_peeruuid[40]; /* UUID of MDS associated with this OST */ + __u32 lsd_osd_index; /* index number of OST/MDT in LOV/LMV */ + __u8 lsd_padding[LR_SERVER_SIZE - 148]; +#endif + + printf("UUID %s\n", lsd.lsd_uuid); + printf("Feature compat=%#x\n", lsd.lsd_feature_compat); + printf("Feature incompat=%#x\n", lsd.lsd_feature_incompat); + printf("Feature rocompat=%#x\n", lsd.lsd_feature_rocompat); + printf("Last transaction %llu\n", (long long)lsd.lsd_last_transno); + printf("target index %u\n", lsd.lsd_osd_index); + + if ((lsd.lsd_feature_compat & OBD_COMPAT_OST) || + (lsd.lsd_feature_incompat & OBD_INCOMPAT_OST)) { + printf("OST, index %d\n", lsd.lsd_osd_index); + } else if ((lsd.lsd_feature_compat & OBD_COMPAT_MDT) || + (lsd.lsd_feature_incompat & OBD_INCOMPAT_MDT)) { + /* We must co-locate so mgs can see old logs. + If user doesn't want this, they can copy the old + logs manually and re-tunefs. */ + printf("MDS, index %d\n", lsd.lsd_osd_index); + } else { + /* If neither is set, we're pre-1.4.6, make a guess. */ + /* Construct debugfs command line. */ + memset(cmd, 0, sizeof(cmd)); + sprintf(cmd, "%s -c -R 'rdump /%s %s' %s", + DEBUGFS, MDT_LOGS_DIR, tmpdir, dev); + + run_command(cmd); + + sprintf(filepnm, "%s/%s", tmpdir, MDT_LOGS_DIR); + if (lsd.lsd_osd_index > 0) { + printf("non-flagged OST, index %d\n", + lsd.lsd_osd_index); + } else { + /* If there's a LOGS dir, it's an MDT */ + if ((ret = access(filepnm, F_OK)) == 0) { + /* Old MDT's are always index 0 + (pre CMD) */ + printf("non-flagged MDS, index 0\n"); + } else { + printf("non-flagged OST, index unknown\n"); + } + } + } + out_close: - fclose(filep); + fclose(filep); out_rmdir: - memset(cmd, 0, sizeof(cmd)); - sprintf(cmd, "rm -rf %s", tmpdir); - run_command(cmd); - return ret; + memset(cmd, 0, sizeof(cmd)); + sprintf(cmd, "rm -rf %s", tmpdir); + run_command(cmd); + return ret; }