X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Fmds%2Fmds_fs.c;h=7194c3bbe691d4e0329ac47a1004de2d067e8381;hb=32b4812a6afd1e71a3576835d445622d38cdafd1;hp=56346ca007a93f117a1b38752b29961e2b0e25df;hpb=a2a0746305449dbd925879b14dc2c0d6040bb8bf;p=fs%2Flustre-release.git diff --git a/lustre/mds/mds_fs.c b/lustre/mds/mds_fs.c index 56346ca..7194c3b 100644 --- a/lustre/mds/mds_fs.c +++ b/lustre/mds/mds_fs.c @@ -23,7 +23,9 @@ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ -#define EXPORT_SYMTAB +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif #define DEBUG_SUBSYSTEM S_MDS #include @@ -46,6 +48,7 @@ #define MDS_MAX_CLIENT_WORDS (MDS_MAX_CLIENTS / sizeof(unsigned long)) #define LAST_RCVD "last_rcvd" +#define LOV_OBJID "lov_objid" /* Add client data to the MDS. We use a bitmap to locate a free space * in the last_rcvd file if cl_off is -1 (i.e. a new client). @@ -57,11 +60,12 @@ int mds_client_add(struct obd_device *obd, struct mds_obd *mds, { unsigned long *bitmap = mds->mds_client_bitmap; int new_client = (cl_idx == -1); + ENTRY; LASSERT(bitmap != NULL); /* XXX if mcd_uuid were a real obd_uuid, I could use obd_uuid_equals */ - if (!strcmp(med->med_mcd->mcd_uuid, "OBD_CLASS_UUID")) + if (!strcmp(med->med_mcd->mcd_uuid, obd->obd_uuid.uuid)) RETURN(0); /* the bitmap operations can handle cl_idx > sizeof(long) * 8, so @@ -75,8 +79,6 @@ int mds_client_add(struct obd_device *obd, struct mds_obd *mds, return -ENOMEM; } if (test_and_set_bit(cl_idx, bitmap)) { - CERROR("MDS client %d: found bit is set in bitmap\n", - cl_idx); cl_idx = find_next_zero_bit(bitmap, MDS_MAX_CLIENTS, cl_idx); goto repeat; @@ -89,55 +91,26 @@ int mds_client_add(struct obd_device *obd, struct mds_obd *mds, } } - CDEBUG(D_INFO, "client at index %d with UUID '%s' added\n", + CDEBUG(D_INFO, "client at idx %d with UUID '%s' added\n", cl_idx, med->med_mcd->mcd_uuid); med->med_idx = cl_idx; - med->med_off = MDS_LR_CLIENT_START + (cl_idx * MDS_LR_CLIENT_SIZE); + med->med_off = le32_to_cpu(mds->mds_server_data->msd_client_start) + + (cl_idx * le16_to_cpu(mds->mds_server_data->msd_client_size)); if (new_client) { - struct obd_run_ctxt saved; + struct lvfs_run_ctxt saved; loff_t off = med->med_off; - ssize_t written; - void *handle; - - push_ctxt(&saved, &mds->mds_ctxt, NULL); - /* We need to start a transaction here first, to avoid a - * possible ordering deadlock on last_rcvd->i_sem and the - * journal lock. In most places we start the journal handle - * first (because we do compound transactions), and then - * later do the write into last_rcvd, which gets i_sem. - * - * Without this transaction, clients connecting at the same - * time other MDS operations are ongoing get last_rcvd->i_sem - * first (in generic_file_write()) and start the journal - * transaction afterwards, and can deadlock with other ops. - * - * We use FSFILT_OP_SETATTR because it is smallest, but all - * ops include enough space for the last_rcvd update so we - * could use any of them, or maybe an FSFILT_OP_NONE is best? - */ - handle = fsfilt_start(obd,mds->mds_rcvd_filp->f_dentry->d_inode, - FSFILT_OP_SETATTR, NULL); - if (IS_ERR(handle)) { - written = PTR_ERR(handle); - CERROR("unable to start transaction: rc %d\n", - (int)written); - } else { - written = fsfilt_write_record(obd, mds->mds_rcvd_filp, - (char *)med->med_mcd, - sizeof(*med->med_mcd), - &off); - fsfilt_commit(obd,mds->mds_rcvd_filp->f_dentry->d_inode, - handle, 0); - } - pop_ctxt(&saved, &mds->mds_ctxt, NULL); + struct file *file = mds->mds_rcvd_filp; + int rc; - if (written != sizeof(*med->med_mcd)) { - if (written < 0) - RETURN(written); - RETURN(-EIO); - } + push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + rc = fsfilt_write_record(obd, file, med->med_mcd, + sizeof(*med->med_mcd), &off, 1); + pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + + if (rc) + return rc; CDEBUG(D_INFO, "wrote client mcd at idx %u off %llu (len %u)\n", med->med_idx, med->med_off, (unsigned int)sizeof(*med->med_mcd)); @@ -145,47 +118,52 @@ int mds_client_add(struct obd_device *obd, struct mds_obd *mds, return 0; } -int mds_client_free(struct obd_export *exp) +int mds_client_free(struct obd_export *exp, int clear_client) { struct mds_export_data *med = &exp->exp_mds_data; struct mds_obd *mds = &exp->exp_obd->u.mds; struct obd_device *obd = exp->exp_obd; struct mds_client_data zero_mcd; - struct obd_run_ctxt saved; - int written; + struct lvfs_run_ctxt saved; + int rc; unsigned long *bitmap = mds->mds_client_bitmap; - LASSERT(bitmap); if (!med->med_mcd) RETURN(0); /* XXX if mcd_uuid were a real obd_uuid, I could use obd_uuid_equals */ - if (!strcmp(med->med_mcd->mcd_uuid, "OBD_CLASS_UUID")) + if (!strcmp(med->med_mcd->mcd_uuid, obd->obd_uuid.uuid)) GOTO(free_and_out, 0); - CDEBUG(D_INFO, "freeing client at index %u (%lld)with UUID '%s'\n", + CDEBUG(D_INFO, "freeing client at idx %u (%lld)with UUID '%s'\n", med->med_idx, med->med_off, med->med_mcd->mcd_uuid); - if (!test_and_clear_bit(med->med_idx, bitmap)) { + LASSERT(bitmap); + + /* Clear the bit _after_ zeroing out the client so we don't + race with mds_client_add and zero out new clients.*/ + if (!test_bit(med->med_idx, bitmap)) { CERROR("MDS client %u: bit already clear in bitmap!!\n", med->med_idx); LBUG(); } - memset(&zero_mcd, 0, sizeof zero_mcd); - push_ctxt(&saved, &mds->mds_ctxt, NULL); - written = fsfilt_write_record(obd, mds->mds_rcvd_filp, - (char *)&zero_mcd, sizeof(zero_mcd), - &med->med_off); - pop_ctxt(&saved, &mds->mds_ctxt, NULL); + if (clear_client) { + memset(&zero_mcd, 0, sizeof zero_mcd); + push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + rc = fsfilt_write_record(obd, mds->mds_rcvd_filp, &zero_mcd, + sizeof(zero_mcd), &med->med_off, 1); + pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - if (written != sizeof(zero_mcd)) { - CERROR("error zeroing out client %s index %d in %s: %d\n", - med->med_mcd->mcd_uuid, med->med_idx, LAST_RCVD, - written); - } else { - CDEBUG(D_INFO, "zeroed out disconnecting client %s at off %d\n", - med->med_mcd->mcd_uuid, med->med_idx); + CDEBUG(rc == 0 ? D_INFO : D_ERROR, + "zeroing out client %s idx %u in %s rc %d\n", + med->med_mcd->mcd_uuid, med->med_idx, LAST_RCVD, rc); + } + + if (!test_and_clear_bit(med->med_idx, bitmap)) { + CERROR("MDS client %u: bit already clear in bitmap!!\n", + med->med_idx); + LBUG(); } free_and_out: @@ -210,20 +188,22 @@ static int mds_read_last_rcvd(struct obd_device *obd, struct file *file) struct mds_server_data *msd; struct mds_client_data *mcd = NULL; loff_t off = 0; - int cl_idx; unsigned long last_rcvd_size = file->f_dentry->d_inode->i_size; - __u64 last_transno = 0; __u64 mount_count; - int rc = 0; + int cl_idx, rc = 0; + ENTRY; - LASSERT(sizeof(struct mds_client_data) == MDS_LR_CLIENT_SIZE); - LASSERT(sizeof(struct mds_server_data) <= MDS_LR_SERVER_SIZE); + /* ensure padding in the struct is the correct size */ + LASSERT(offsetof(struct mds_server_data, msd_padding) + + sizeof(msd->msd_padding) == MDS_LR_SERVER_SIZE); + LASSERT(offsetof(struct mds_client_data, mcd_padding) + + sizeof(mcd->mcd_padding) == MDS_LR_CLIENT_SIZE); - OBD_ALLOC(msd, sizeof(*msd)); + OBD_ALLOC_WAIT(msd, sizeof(*msd)); if (!msd) RETURN(-ENOMEM); - OBD_ALLOC(mds->mds_client_bitmap, + OBD_ALLOC_WAIT(mds->mds_client_bitmap, MDS_MAX_CLIENT_WORDS * sizeof(unsigned long)); if (!mds->mds_client_bitmap) { OBD_FREE(msd, sizeof(*msd)); @@ -234,51 +214,49 @@ static int mds_read_last_rcvd(struct obd_device *obd, struct file *file) if (last_rcvd_size == 0) { CWARN("%s: initializing new %s\n", obd->obd_name, LAST_RCVD); + memcpy(msd->msd_uuid, obd->obd_uuid.uuid,sizeof(msd->msd_uuid)); + msd->msd_last_transno = 0; + mount_count = msd->msd_mount_count = 0; msd->msd_server_size = cpu_to_le32(MDS_LR_SERVER_SIZE); msd->msd_client_start = cpu_to_le32(MDS_LR_CLIENT_START); msd->msd_client_size = cpu_to_le16(MDS_LR_CLIENT_SIZE); - - RETURN(0); - } - - rc = fsfilt_read_record(obd, file, (char *)msd, sizeof(*msd), &off); - - if (rc != sizeof(*msd)) { - CERROR("error reading MDS %s: rc = %d\n", LAST_RCVD,rc); - if (rc > 0) - rc = -EIO; - GOTO(err_msd, rc); + msd->msd_feature_rocompat = cpu_to_le32(MDS_ROCOMPAT_LOVOBJID); + } else { + rc = fsfilt_read_record(obd, file, msd, sizeof(*msd), &off); + if (rc) { + CERROR("error reading MDS %s: rc = %d\n", LAST_RCVD, rc); + GOTO(err_msd, rc); + } + if (strcmp(msd->msd_uuid, obd->obd_uuid.uuid) != 0) { + CERROR("OBD UUID %s does not match last_rcvd UUID %s\n", + obd->obd_uuid.uuid, msd->msd_uuid); + GOTO(err_msd, rc = -EINVAL); + } + mount_count = le64_to_cpu(msd->msd_mount_count); } - if (!msd->msd_server_size) - msd->msd_server_size = cpu_to_le32(MDS_LR_SERVER_SIZE); - if (!msd->msd_client_start) - msd->msd_client_start = cpu_to_le32(MDS_LR_CLIENT_START); - if (!msd->msd_client_size) - msd->msd_client_size = cpu_to_le16(MDS_LR_CLIENT_SIZE); - - if (msd->msd_feature_incompat) { + if (msd->msd_feature_incompat & ~cpu_to_le32(MDS_INCOMPAT_SUPP)) { CERROR("unsupported incompat feature %x\n", - le32_to_cpu(msd->msd_feature_incompat)); + le32_to_cpu(msd->msd_feature_incompat) & + ~MDS_INCOMPAT_SUPP); GOTO(err_msd, rc = -EINVAL); } - if (msd->msd_feature_rocompat) { + /* XXX updating existing b_devel fs only, can be removed in future */ + msd->msd_feature_rocompat = cpu_to_le32(MDS_ROCOMPAT_LOVOBJID); + if (msd->msd_feature_rocompat & ~cpu_to_le32(MDS_ROCOMPAT_SUPP)) { CERROR("unsupported read-only feature %x\n", - le32_to_cpu(msd->msd_feature_rocompat)); + le32_to_cpu(msd->msd_feature_rocompat) & + ~MDS_ROCOMPAT_SUPP); /* Do something like remount filesystem read-only */ GOTO(err_msd, rc = -EINVAL); } - last_transno = le64_to_cpu(msd->msd_last_transno); - mds->mds_last_transno = last_transno; - - mount_count = le64_to_cpu(msd->msd_mount_count); - mds->mds_mount_count = mount_count; + mds->mds_last_transno = le64_to_cpu(msd->msd_last_transno); CDEBUG(D_INODE, "%s: server last_transno: "LPU64"\n", - obd->obd_name, last_transno); + obd->obd_name, mds->mds_last_transno); CDEBUG(D_INODE, "%s: server mount_count: "LPU64"\n", - obd->obd_name, mount_count); + obd->obd_name, mount_count + 1); CDEBUG(D_INODE, "%s: server data size: %u\n", obd->obd_name, le32_to_cpu(msd->msd_server_size)); CDEBUG(D_INODE, "%s: per-client data start: %u\n", @@ -288,34 +266,35 @@ static int mds_read_last_rcvd(struct obd_device *obd, struct file *file) CDEBUG(D_INODE, "%s: last_rcvd size: %lu\n", obd->obd_name, last_rcvd_size); CDEBUG(D_INODE, "%s: last_rcvd clients: %lu\n", obd->obd_name, - (last_rcvd_size - MDS_LR_CLIENT_START) / MDS_LR_CLIENT_SIZE); + last_rcvd_size <= le32_to_cpu(msd->msd_client_start) ? 0 : + (last_rcvd_size - le32_to_cpu(msd->msd_client_start)) / + le16_to_cpu(msd->msd_client_size)); - /* When we do a clean FILTER shutdown, we save the last_transno into + /* When we do a clean MDS shutdown, we save the last_transno into * the header. If we find clients with higher last_transno values * then those clients may need recovery done. */ - for (cl_idx = 0; off < last_rcvd_size; cl_idx++) { + for (cl_idx = 0, off = le32_to_cpu(msd->msd_client_start); + off < last_rcvd_size; cl_idx++) { __u64 last_transno; - int mount_age; + struct obd_export *exp; + struct mds_export_data *med; if (!mcd) { - OBD_ALLOC(mcd, sizeof(*mcd)); + OBD_ALLOC_WAIT(mcd, sizeof(*mcd)); if (!mcd) - GOTO(err_msd, rc = -ENOMEM); + GOTO(err_client, rc = -ENOMEM); } - /* Don't assume off is incremented properly, in case - * sizeof(fsd) isn't the same as fsd->fsd_client_size. - */ + /* Don't assume off is incremented properly by + * fsfilt_read_record(), in case sizeof(*mcd) + * isn't the same as msd->msd_client_size. */ off = le32_to_cpu(msd->msd_client_start) + cl_idx * le16_to_cpu(msd->msd_client_size); - rc = fsfilt_read_record(obd, file, (char *)mcd, - sizeof(*mcd), &off); - if (rc != sizeof(*mcd)) { - CERROR("error reading MDS %s offset %d: rc = %d\n", - LAST_RCVD, cl_idx, rc); - if (rc > 0) /* XXX fatal error or just abort reading? */ - rc = -EIO; - break; + rc = fsfilt_read_record(obd, file, mcd, sizeof(*mcd), &off); + if (rc) { + CERROR("error reading MDS %s idx %d, off %llu: rc %d\n", + LAST_RCVD, cl_idx, off, rc); + break; /* read error shouldn't cause startup to fail */ } if (mcd->mcd_uuid[0] == '\0') { @@ -329,76 +308,108 @@ static int mds_read_last_rcvd(struct obd_device *obd, struct file *file) /* These exports are cleaned up by mds_disconnect(), so they * need to be set up like real exports as mds_connect() does. */ - mount_age = mount_count - le64_to_cpu(mcd->mcd_mount_count); - if (mount_age < MDS_MOUNT_RECOV) { - struct obd_export *exp = class_new_export(obd); - struct mds_export_data *med; - CERROR("RCVRNG CLIENT uuid: %s off: %d lr: "LPU64 - "srv lr: "LPU64" mnt: "LPU64" last mount: "LPU64 - "\n", mcd->mcd_uuid, cl_idx, - last_transno, le64_to_cpu(msd->msd_last_transno), - le64_to_cpu(mcd->mcd_mount_count), mount_count); - - if (!exp) { - rc = -ENOMEM; - break; - } - - memcpy(&exp->exp_client_uuid.uuid, mcd->mcd_uuid, - sizeof exp->exp_client_uuid.uuid); - med = &exp->exp_mds_data; - med->med_mcd = mcd; - mds_client_add(obd, mds, med, cl_idx); - /* create helper if export init gets more complex */ - INIT_LIST_HEAD(&med->med_open_head); - spin_lock_init(&med->med_open_lock); - - mcd = NULL; - obd->obd_recoverable_clients++; - class_export_put(exp); - } else { - CDEBUG(D_INFO, "discarded client %d, UUID '%s', count " - LPU64"\n", cl_idx, mcd->mcd_uuid, - le64_to_cpu(mcd->mcd_mount_count)); - } - - CDEBUG(D_OTHER, "client at offset %d has last_transno = " - LPU64"\n", cl_idx, last_transno); + CDEBUG(D_HA, "RCVRNG CLIENT uuid: %s idx: %d lr: "LPU64 + " srv lr: "LPU64"\n", mcd->mcd_uuid, cl_idx, + last_transno, le64_to_cpu(msd->msd_last_transno)); + + exp = class_new_export(obd); + if (exp == NULL) + GOTO(err_client, rc = -ENOMEM); + + memcpy(&exp->exp_client_uuid.uuid, mcd->mcd_uuid, + sizeof exp->exp_client_uuid.uuid); + med = &exp->exp_mds_data; + med->med_mcd = mcd; + mds_client_add(obd, mds, med, cl_idx); + /* create helper if export init gets more complex */ + INIT_LIST_HEAD(&med->med_open_head); + spin_lock_init(&med->med_open_lock); + + mcd = NULL; + obd->obd_recoverable_clients++; + obd->obd_max_recoverable_clients++; + class_export_put(exp); + + CDEBUG(D_OTHER, "client at idx %d has last_transno = "LPU64"\n", + cl_idx, last_transno); if (last_transno > mds->mds_last_transno) - mds->mds_last_transno = last_transno; + mds->mds_last_transno = last_transno; } obd->obd_last_committed = mds->mds_last_transno; if (obd->obd_recoverable_clients) { - CERROR("RECOVERY: %d recoverable clients, last_transno " - LPU64"\n", - obd->obd_recoverable_clients, mds->mds_last_transno); - obd->obd_next_recovery_transno = obd->obd_last_committed - + 1; + CWARN("RECOVERY: service %s, %d recoverable clients, " + "last_transno "LPU64"\n", obd->obd_name, + obd->obd_recoverable_clients, mds->mds_last_transno); + obd->obd_next_recovery_transno = obd->obd_last_committed + 1; obd->obd_recovering = 1; } if (mcd) OBD_FREE(mcd, sizeof(*mcd)); + + mds->mds_mount_count = mount_count + 1; + msd->msd_mount_count = cpu_to_le64(mds->mds_mount_count); - return 0; + /* save it, so mount count and last_transno is current */ + rc = mds_update_server_data(obd, 1); + + RETURN(rc); +err_client: + class_disconnect_exports(obd, 0); err_msd: mds_server_free_data(mds); - return rc; + RETURN(rc); } -static int mds_fs_prep(struct obd_device *obd) +static int mds_fs_post_setup(struct obd_device *obd) { struct mds_obd *mds = &obd->u.mds; - struct obd_run_ctxt saved; + struct dentry *de = mds_fid2dentry(mds, &mds->mds_rootfid, NULL); + int rc = 0; + + rc = fsfilt_post_setup(obd); + if (rc) + GOTO(out, rc); + + fsfilt_set_kml_flags(obd, de->d_inode); + fsfilt_set_kml_flags(obd, mds->mds_pending_dir->d_inode); + + fsfilt_set_mds_flags(obd, mds->mds_sb); +out: + l_dput(de); + return rc; +} + +int mds_fs_setup(struct obd_device *obd, struct vfsmount *mnt) +{ + struct mds_obd *mds = &obd->u.mds; + struct lvfs_run_ctxt saved; struct dentry *dentry; struct file *file; int rc; + ENTRY; + + rc = cleanup_group_info(); + if (rc) + RETURN(rc); + + mds->mds_vfsmnt = mnt; + mds->mds_sb = mnt->mnt_root->d_inode->i_sb; + + fsfilt_setup(obd, mds->mds_sb); - push_ctxt(&saved, &mds->mds_ctxt, NULL); - dentry = simple_mkdir(current->fs->pwd, "ROOT", 0755); + OBD_SET_CTXT_MAGIC(&obd->obd_lvfs_ctxt); + obd->obd_lvfs_ctxt.pwdmnt = mnt; + obd->obd_lvfs_ctxt.pwd = mnt->mnt_root; + obd->obd_lvfs_ctxt.fs = get_ds(); + obd->obd_lvfs_ctxt.cb_ops = mds_lvfs_ops; + + /* setup the directory tree */ + push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + dentry = simple_mkdir(current->fs->pwd, "ROOT", 0755, 0); if (IS_ERR(dentry)) { rc = PTR_ERR(dentry); CERROR("cannot create ROOT directory: rc = %d\n", rc); @@ -413,22 +424,27 @@ static int mds_fs_prep(struct obd_device *obd) dentry = lookup_one_len("__iopen__", current->fs->pwd, strlen("__iopen__")); - if (IS_ERR(dentry) || !dentry->d_inode) { - rc = (IS_ERR(dentry)) ? PTR_ERR(dentry): -ENOENT; - CERROR("cannot open iopen FH directory: rc = %d\n", rc); + if (IS_ERR(dentry)) { + rc = PTR_ERR(dentry); + CERROR("cannot lookup __iopen__ directory: rc = %d\n", rc); GOTO(err_pop, rc); } + if (!dentry->d_inode) { + rc = -ENOENT; + CERROR("__iopen__ directory has no inode? rc = %d\n", rc); + GOTO(err_fid, rc); + } mds->mds_fid_de = dentry; - dentry = simple_mkdir(current->fs->pwd, "PENDING", 0777); + dentry = simple_mkdir(current->fs->pwd, "PENDING", 0777, 1); if (IS_ERR(dentry)) { rc = PTR_ERR(dentry); CERROR("cannot create PENDING directory: rc = %d\n", rc); GOTO(err_fid, rc); } mds->mds_pending_dir = dentry; - - dentry = simple_mkdir(current->fs->pwd, "LOGS", 0700); + + dentry = simple_mkdir(current->fs->pwd, "LOGS", 0777, 1); if (IS_ERR(dentry)) { rc = PTR_ERR(dentry); CERROR("cannot create LOGS directory: rc = %d\n", rc); @@ -436,48 +452,76 @@ static int mds_fs_prep(struct obd_device *obd) } mds->mds_logs_dir = dentry; + dentry = simple_mkdir(current->fs->pwd, "OBJECTS", 0777, 1); + if (IS_ERR(dentry)) { + rc = PTR_ERR(dentry); + CERROR("cannot create OBJECTS directory: rc = %d\n", rc); + GOTO(err_logs, rc); + } + mds->mds_objects_dir = dentry; + + dentry = simple_mkdir(current->fs->pwd, "FIDS", 0777, 1); + if (IS_ERR(dentry)) { + rc = PTR_ERR(dentry); + CERROR("cannot create FIDS directory: rc = %d\n", rc); + GOTO(err_fids, rc); + } + mds->mds_fids_dir = dentry; + + /* open and test the last rcvd file */ file = filp_open(LAST_RCVD, O_RDWR | O_CREAT, 0644); if (IS_ERR(file)) { rc = PTR_ERR(file); CERROR("cannot open/create %s file: rc = %d\n", LAST_RCVD, rc); - - GOTO(err_logs, rc = PTR_ERR(file)); + GOTO(err_objects, rc = PTR_ERR(file)); } + mds->mds_rcvd_filp = file; if (!S_ISREG(file->f_dentry->d_inode->i_mode)) { CERROR("%s is not a regular file!: mode = %o\n", LAST_RCVD, file->f_dentry->d_inode->i_mode); - GOTO(err_filp, rc = -ENOENT); - } - - rc = fsfilt_journal_data(obd, file); - if (rc) { - CERROR("cannot journal data on %s: rc = %d\n", LAST_RCVD, rc); - GOTO(err_filp, rc); + GOTO(err_last_rcvd, rc = -ENOENT); } rc = mds_read_last_rcvd(obd, file); if (rc) { CERROR("cannot read %s: rc = %d\n", LAST_RCVD, rc); - GOTO(err_client, rc); + GOTO(err_last_rcvd, rc); } - mds->mds_rcvd_filp = file; -#ifdef I_SKIP_PDFLUSH - /* - * we need this to protect from deadlock - * pdflush vs. lustre_fwrite() - */ - file->f_dentry->d_inode->i_flags |= I_SKIP_PDFLUSH; -#endif -err_pop: - pop_ctxt(&saved, &mds->mds_ctxt, NULL); + /* open and test the lov objd file */ + file = filp_open(LOV_OBJID, O_RDWR | O_CREAT, 0644); + if (IS_ERR(file)) { + rc = PTR_ERR(file); + CERROR("cannot open/create %s file: rc = %d\n", LOV_OBJID, rc); + GOTO(err_client, rc = PTR_ERR(file)); + } + mds->mds_lov_objid_filp = file; + if (!S_ISREG(file->f_dentry->d_inode->i_mode)) { + CERROR("%s is not a regular file!: mode = %o\n", LOV_OBJID, + file->f_dentry->d_inode->i_mode); + GOTO(err_lov_objid, rc = -ENOENT); + } +err_pop: + if (!rc) { + rc = mds_fs_post_setup(obd); + if (rc) + CERROR("can not post setup fsfilt\n"); + } + pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); return rc; +err_lov_objid: + if (mds->mds_lov_objid_filp && filp_close(mds->mds_lov_objid_filp, 0)) + CERROR("can't close %s after error\n", LOV_OBJID); err_client: class_disconnect_exports(obd, 0); -err_filp: - if (filp_close(file, 0)) +err_last_rcvd: + if (mds->mds_rcvd_filp && filp_close(mds->mds_rcvd_filp, 0)) CERROR("can't close %s after error\n", LAST_RCVD); +err_fids: + dput(mds->mds_fids_dir); +err_objects: + dput(mds->mds_objects_dir); err_logs: dput(mds->mds_logs_dir); err_pending: @@ -487,24 +531,17 @@ err_fid: goto err_pop; } -int mds_fs_setup(struct obd_device *obd, struct vfsmount *mnt) +static int mds_fs_post_cleanup(struct obd_device *obd) { - struct mds_obd *mds = &obd->u.mds; - ENTRY; - - mds->mds_vfsmnt = mnt; - - OBD_SET_CTXT_MAGIC(&mds->mds_ctxt); - mds->mds_ctxt.pwdmnt = mnt; - mds->mds_ctxt.pwd = mnt->mnt_root; - mds->mds_ctxt.fs = get_ds(); - RETURN(mds_fs_prep(obd)); + int rc = 0; + rc = fsfilt_post_cleanup(obd); + return rc; } int mds_fs_cleanup(struct obd_device *obd, int flags) { struct mds_obd *mds = &obd->u.mds; - struct obd_run_ctxt saved; + struct lvfs_run_ctxt saved; int rc = 0; if (flags & OBD_OPT_FAILOVER) @@ -514,13 +551,27 @@ int mds_fs_cleanup(struct obd_device *obd, int flags) class_disconnect_exports(obd, flags); /* cleans up client info too */ mds_server_free_data(mds); - push_ctxt(&saved, &mds->mds_ctxt, NULL); + push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); if (mds->mds_rcvd_filp) { rc = filp_close(mds->mds_rcvd_filp, 0); mds->mds_rcvd_filp = NULL; if (rc) CERROR("%s file won't close, rc=%d\n", LAST_RCVD, rc); } + if (mds->mds_lov_objid_filp) { + rc = filp_close(mds->mds_lov_objid_filp, 0); + mds->mds_lov_objid_filp = NULL; + if (rc) + CERROR("%s file won't close, rc=%d\n", LOV_OBJID, rc); + } + if (mds->mds_fids_dir != NULL) { + l_dput(mds->mds_fids_dir); + mds->mds_fids_dir = NULL; + } + if (mds->mds_objects_dir != NULL) { + l_dput(mds->mds_objects_dir); + mds->mds_objects_dir = NULL; + } if (mds->mds_logs_dir) { l_dput(mds->mds_logs_dir); mds->mds_logs_dir = NULL; @@ -529,239 +580,148 @@ int mds_fs_cleanup(struct obd_device *obd, int flags) l_dput(mds->mds_pending_dir); mds->mds_pending_dir = NULL; } - pop_ctxt(&saved, &mds->mds_ctxt, NULL); + rc = mds_fs_post_cleanup(obd); + + pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); shrink_dcache_parent(mds->mds_fid_de); dput(mds->mds_fid_de); return rc; } -/* This is a callback from the llog_* functions. - * Assumes caller has already pushed us into the kernel context. */ -int mds_log_close(struct llog_handle *cathandle, struct llog_handle *loghandle) +/* Creates an object with the same name as its fid. Because this is not at all + * performance sensitive, it is accomplished by creating a file, checking the + * fid, and renaming it. */ +int mds_obd_create(struct obd_export *exp, struct obdo *oa, + struct lov_stripe_md **ea, struct obd_trans_info *oti) { - struct llog_object_hdr *llh = loghandle->lgh_hdr; - struct mds_obd *mds = &cathandle->lgh_obd->u.mds; - struct dentry *dchild = NULL; - int rc; + struct mds_obd *mds = &exp->exp_obd->u.mds; + struct inode *parent_inode = mds->mds_objects_dir->d_inode; + unsigned int tmpname = ll_insecure_random_int(); + struct file *filp; + struct dentry *new_child; + struct lvfs_run_ctxt saved; + char fidname[LL_FID_NAMELEN]; + void *handle; + int rc = 0, err, namelen; ENTRY; - /* If we are going to delete this log, grab a ref before we close - * it so we don't have to immediately do another lookup. - */ - if (llh->llh_hdr.lth_type != LLOG_CATALOG_MAGIC && llh->llh_count == 0){ - CDEBUG(D_INODE, "deleting log file "LPX64":%x\n", - loghandle->lgh_cookie.lgc_lgl.lgl_oid, - loghandle->lgh_cookie.lgc_lgl.lgl_ogen); - down(&mds->mds_logs_dir->d_inode->i_sem); - dchild = dget(loghandle->lgh_file->f_dentry); - llog_delete_log(cathandle, loghandle); - } else { - CDEBUG(D_INODE, "closing log file "LPX64":%x\n", - loghandle->lgh_cookie.lgc_lgl.lgl_oid, - loghandle->lgh_cookie.lgc_lgl.lgl_ogen); + push_ctxt(&saved, &exp->exp_obd->obd_lvfs_ctxt, NULL); + + sprintf(fidname, "OBJECTS/%u", tmpname); + filp = filp_open(fidname, O_CREAT | O_EXCL, 0644); + if (IS_ERR(filp)) { + rc = PTR_ERR(filp); + if (rc == -EEXIST) { + CERROR("impossible object name collision %u\n", + tmpname); + LBUG(); + } + CERROR("error creating tmp object %u: rc %d\n", tmpname, rc); + GOTO(out_pop, rc); } - rc = filp_close(loghandle->lgh_file, 0); - - llog_free_handle(loghandle); /* also removes loghandle from list */ + LASSERT(mds->mds_objects_dir == filp->f_dentry->d_parent); - if (dchild) { - int err = vfs_unlink(mds->mds_logs_dir->d_inode, dchild); - if (err) { - CERROR("error unlinking empty log %*s: rc %d\n", - dchild->d_name.len, dchild->d_name.name, err); - if (!rc) - rc = err; - } - l_dput(dchild); - up(&mds->mds_logs_dir->d_inode->i_sem); - } - RETURN(rc); -} + oa->o_id = filp->f_dentry->d_inode->i_ino; + oa->o_generation = filp->f_dentry->d_inode->i_generation; + namelen = ll_fid2str(fidname, oa->o_id, oa->o_generation); -/* This is a callback from the llog_* functions. - * Assumes caller has already pushed us into the kernel context. */ -struct llog_handle *mds_log_open(struct obd_device *obd, - struct llog_cookie *logcookie) -{ - struct ll_fid fid = { .id = logcookie->lgc_lgl.lgl_oid, - .generation = logcookie->lgc_lgl.lgl_ogen, - .f_type = S_IFREG }; - struct llog_handle *loghandle; - struct dentry *dchild; - int rc; - ENTRY; + down(&parent_inode->i_sem); + new_child = lookup_one_len(fidname, mds->mds_objects_dir, namelen); - loghandle = llog_alloc_handle(); - if (loghandle == NULL) - RETURN(ERR_PTR(-ENOMEM)); - - down(&obd->u.mds.mds_logs_dir->d_inode->i_sem); - dchild = mds_fid2dentry(&obd->u.mds, &fid, NULL); - up(&obd->u.mds.mds_logs_dir->d_inode->i_sem); - if (IS_ERR(dchild)) { - rc = PTR_ERR(dchild); - CERROR("error looking up log file "LPX64":%x: rc %d\n", - fid.id, fid.generation, rc); - GOTO(out, rc); + if (IS_ERR(new_child)) { + CERROR("getting neg dentry for obj rename: %d\n", rc); + GOTO(out_close, rc = PTR_ERR(new_child)); } - - if (dchild->d_inode == NULL) { - rc = -ENOENT; - CERROR("nonexistent log file "LPX64":%x: rc %d\n", - fid.id, fid.generation, rc); - GOTO(out_put, rc); - } - - /* dentry_open does a dput(de) and mntput(mds->mds_vfsmnt) on error */ - mntget(obd->u.mds.mds_vfsmnt); - loghandle->lgh_file = dentry_open(dchild, obd->u.mds.mds_vfsmnt, - O_RDWR | O_LARGEFILE); - if (IS_ERR(loghandle->lgh_file)) { - rc = PTR_ERR(loghandle->lgh_file); - CERROR("error opening logfile "LPX64":%x: rc %d\n", - fid.id, fid.generation, rc); - GOTO(out, rc); + if (new_child->d_inode != NULL) { + CERROR("impossible non-negative obj dentry " LPU64":%u!\n", + oa->o_id, oa->o_generation); + LBUG(); } - memcpy(&loghandle->lgh_cookie, logcookie, sizeof(*logcookie)); - loghandle->lgh_log_create = mds_log_create; - loghandle->lgh_log_open = mds_log_open; - loghandle->lgh_log_close = mds_log_close; - loghandle->lgh_obd = obd; - - RETURN(loghandle); - -out_put: - l_dput(dchild); -out: - llog_free_handle(loghandle); - return ERR_PTR(rc); -} -/* This is a callback from the llog_* functions. - * Assumes caller has already pushed us into the kernel context. */ -struct llog_handle *mds_log_create(struct obd_device *obd) -{ - char logbuf[24], *logname; /* logSSSSSSSSSS.count */ - struct llog_handle *loghandle; - int rc, open_flags = O_RDWR | O_CREAT | O_LARGEFILE; - ENTRY; + handle = fsfilt_start(exp->exp_obd, mds->mds_objects_dir->d_inode, + FSFILT_OP_RENAME, NULL); + if (IS_ERR(handle)) + GOTO(out_dput, rc = PTR_ERR(handle)); - loghandle = llog_alloc_handle(); - if (!loghandle) - RETURN(ERR_PTR(-ENOMEM)); - -retry: - if (!obd->u.mds.mds_catalog) { - logname = "LOGS/catalog"; - } else { - sprintf(logbuf, "LOGS/log%lu.%u\n", - CURRENT_SECONDS, obd->u.mds.mds_catalog->lgh_index++); - open_flags |= O_EXCL; - logname = logbuf; - } - loghandle->lgh_file = filp_open(logname, open_flags, 0644); - if (IS_ERR(loghandle->lgh_file)) { - rc = PTR_ERR(loghandle->lgh_file); - if (rc == -EEXIST) { - CDEBUG(D_HA, "collision in logfile %s creation\n", - logname); - obd->u.mds.mds_catalog->lgh_index++; - goto retry; - } - CERROR("error opening/creating %s: rc %d\n", logname, rc); - GOTO(out_handle, rc); + lock_kernel(); + rc = vfs_rename(mds->mds_objects_dir->d_inode, filp->f_dentry, + mds->mds_objects_dir->d_inode, new_child); + unlock_kernel(); + if (rc) + CERROR("error renaming new object "LPU64":%u: rc %d\n", + oa->o_id, oa->o_generation, rc); + + err = fsfilt_commit(exp->exp_obd, mds->mds_sb, + mds->mds_objects_dir->d_inode, handle, 0); + if (!err) { + oa->o_gr = FILTER_GROUP_FIRST_MDS + mds->mds_num; + oa->o_valid |= OBD_MD_FLID | OBD_MD_FLGENER | OBD_MD_FLGROUP; } - - loghandle->lgh_cookie.lgc_lgl.lgl_oid = - loghandle->lgh_file->f_dentry->d_inode->i_ino; - loghandle->lgh_cookie.lgc_lgl.lgl_ogen = - loghandle->lgh_file->f_dentry->d_inode->i_generation; - loghandle->lgh_log_create = mds_log_create; - loghandle->lgh_log_open = mds_log_open; - loghandle->lgh_log_close = mds_log_close; - loghandle->lgh_obd = obd; - - RETURN(loghandle); - -out_handle: - llog_free_handle(loghandle); - return ERR_PTR(rc); + else if (!rc) + rc = err; +out_dput: + dput(new_child); +out_close: + up(&parent_inode->i_sem); + err = filp_close(filp, 0); + if (err) { + CERROR("closing tmpfile %u: rc %d\n", tmpname, rc); + if (!rc) + rc = err; + } +out_pop: + pop_ctxt(&saved, &exp->exp_obd->obd_lvfs_ctxt, NULL); + RETURN(rc); } -struct llog_handle *mds_get_catalog(struct obd_device *obd) +int mds_obd_destroy(struct obd_export *exp, struct obdo *oa, + struct lov_stripe_md *ea, struct obd_trans_info *oti) { - struct mds_server_data *msd = obd->u.mds.mds_server_data; - struct obd_run_ctxt saved; - struct llog_handle *cathandle = NULL; - int rc = 0; + struct mds_obd *mds = &exp->exp_obd->u.mds; + struct inode *parent_inode = mds->mds_objects_dir->d_inode; + struct obd_device *obd = exp->exp_obd; + struct lvfs_run_ctxt saved; + char fidname[LL_FID_NAMELEN]; + struct dentry *de; + void *handle; + int err, namelen, rc = 0; ENTRY; - push_ctxt(&saved, &obd->u.mds.mds_ctxt, NULL); - - if (msd->msd_catalog_oid) { - struct llog_cookie catcookie; - - catcookie.lgc_lgl.lgl_oid = le64_to_cpu(msd->msd_catalog_oid); - catcookie.lgc_lgl.lgl_ogen = le32_to_cpu(msd->msd_catalog_ogen); - cathandle = mds_log_open(obd, &catcookie); - if (IS_ERR(cathandle)) { - CERROR("error opening catalog "LPX64":%x: rc %d\n", - catcookie.lgc_lgl.lgl_oid, - catcookie.lgc_lgl.lgl_ogen, - (int)PTR_ERR(cathandle)); - msd->msd_catalog_oid = 0; - msd->msd_catalog_ogen = 0; - } - /* ORPHANS FIXME: compare catalog UUID to msd_peeruuid */ - } + push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - if (!msd->msd_catalog_oid) { - struct llog_logid *lgl; + namelen = ll_fid2str(fidname, oa->o_id, oa->o_generation); - cathandle = mds_log_create(obd); - if (IS_ERR(cathandle)) { - CERROR("error creating new catalog: rc %d\n", - (int)PTR_ERR(cathandle)); - GOTO(out, cathandle); - } - lgl = &cathandle->lgh_cookie.lgc_lgl; - msd->msd_catalog_oid = cpu_to_le64(lgl->lgl_oid); - msd->msd_catalog_ogen = cpu_to_le32(lgl->lgl_ogen); - rc = mds_update_server_data(obd); - if (rc) { - CERROR("error writing new catalog to disk: rc %d\n",rc); - GOTO(out_handle, rc); - } + down(&parent_inode->i_sem); + de = lookup_one_len(fidname, mds->mds_objects_dir, namelen); + if (de == NULL || de->d_inode == NULL) { + CERROR("destroying non-existent object "LPU64" %s\n", + oa->o_id, fidname); + GOTO(out_dput, rc = IS_ERR(de) ? PTR_ERR(de) : -ENOENT); } - rc = llog_init_catalog(cathandle, &obd->u.mds.mds_osc_uuid); - -out: - pop_ctxt(&saved, &obd->u.mds.mds_ctxt, NULL); - RETURN(cathandle); - -out_handle: - mds_log_close(cathandle, cathandle); - cathandle = ERR_PTR(rc); - goto out; - -} - -void mds_put_catalog(struct llog_handle *cathandle) -{ - struct llog_handle *loghandle, *n; - int rc; - ENTRY; - - list_for_each_entry_safe(loghandle, n, &cathandle->lgh_list, lgh_list) - mds_log_close(cathandle, loghandle); - - rc = filp_close(cathandle->lgh_file, 0); - if (rc) - CERROR("error closing catalog: rc %d\n", rc); - - llog_free_handle(cathandle); - EXIT; + /* Stripe count is 1 here since this is some MDS specific stuff + that is unlinked, not spanned across multiple OSTs */ + handle = fsfilt_start_log(obd, mds->mds_objects_dir->d_inode, + FSFILT_OP_UNLINK, oti, 1); + + if (IS_ERR(handle)) + GOTO(out_dput, rc = PTR_ERR(handle)); + + rc = vfs_unlink(mds->mds_objects_dir->d_inode, de); + if (rc) + CERROR("error destroying object "LPU64":%u: rc %d\n", + oa->o_id, oa->o_generation, rc); + + err = fsfilt_commit(obd, mds->mds_sb, mds->mds_objects_dir->d_inode, + handle, 0); + if (err && !rc) + rc = err; +out_dput: + if (de != NULL) + l_dput(de); + up(&parent_inode->i_sem); + pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + RETURN(rc); }