Whamcloud - gitweb
LU-1842 osp: osp-on-ost device
authorNiu Yawei <niu@whamcloud.com>
Fri, 21 Sep 2012 08:54:54 +0000 (04:54 -0400)
committerOleg Drokin <green@whamcloud.com>
Tue, 2 Oct 2012 06:10:36 +0000 (02:10 -0400)
Setup osp-on-ost device for each OST to maintain the connection
to MDT0, quota slave & ofd will send quota or fid sequence request
over this connection.

We also setup one such device on MDT0 for now, it'll be used by
the quota slave on MDT0, once the DNE code landed, this device
(on MDT0) could be replaced by normal osp device.

Signed-off-by: Wang Di <di.wang@whamcloud.com>
Signed-off-by: Niu Yawei <niu@whamcloud.com>
Change-Id: I2229cec1fa206dcce65a4cceae032566f4c20901
Reviewed-on: http://review.whamcloud.com/4068
Reviewed-by: Johann Lombardi <johann.lombardi@intel.com>
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
14 files changed:
lustre/include/lustre_log.h
lustre/include/obd.h
lustre/include/obd_class.h
lustre/ldlm/ldlm_lib.c
lustre/llite/llite_lib.c
lustre/mgc/mgc_request.c
lustre/obdclass/lustre_peer.c
lustre/obdclass/obd_config.c
lustre/obdclass/obd_mount.c
lustre/osp/Makefile.in
lustre/osp/osp_dev.c
lustre/osp/osp_internal.h
lustre/osp/osp_ost.c [new file with mode: 0644]
lustre/tests/test-framework.sh

index 8b5b98b..b62255f 100644 (file)
@@ -116,8 +116,6 @@ static inline void fid_to_logid(struct lu_fid *fid, struct llog_logid *id)
 struct llog_handle;
 
 /* llog.c  -  general API */
-typedef int (*llog_cb_t)(const struct lu_env *env, struct llog_handle *lgh,
-                        struct llog_rec_hdr *rec, void *data);
 int llog_init_handle(const struct lu_env *env, struct llog_handle *handle,
                     int flags, struct obd_uuid *uuid);
 int llog_process(const struct lu_env *env, struct llog_handle *loghandle,
index 3b9fdf6..48ec59e 100644 (file)
@@ -856,6 +856,33 @@ struct niobuf_local {
 /* Don't conflict with on-wire flags OBD_BRW_WRITE, etc */
 #define N_LOCAL_TEMP_PAGE 0x10000000
 
+static inline int is_osp_on_ost(char *name)
+{
+       char   *ptr;
+
+       ptr = strrchr(name, '-');
+       if (ptr == NULL) {
+               CERROR("%s is not a obdname\n", name);
+               return 0;
+       }
+
+       if (strncmp(ptr + 1, "OST", 3) != 0 && strncmp(ptr + 1, "MDT", 3) != 0)
+               return 0;
+
+       /* match the "-osp" */
+       if (ptr - name < strlen(LUSTRE_OSP_NAME) + 1)
+               return 0;
+
+       ptr -= (strlen(LUSTRE_OSP_NAME) + 1);
+       if (*ptr != '-')
+               return 0;
+
+       if (strncmp(ptr + 1, LUSTRE_OSP_NAME, strlen(LUSTRE_OSP_NAME)) != 0)
+               return 0;
+
+       return 1;
+}
+
 struct obd_trans_info {
         __u64                    oti_transno;
         __u64                    oti_xid;
index f5522ef..c22caa8 100644 (file)
@@ -124,6 +124,10 @@ int kuc_ispayload(void *p);
 void *kuc_alloc(int payload_len, int transport, int type);
 void kuc_free(void *p, int payload_len);
 
+struct llog_handle;
+struct llog_rec_hdr;
+typedef int (*llog_cb_t)(const struct lu_env *, struct llog_handle *,
+                        struct llog_rec_hdr *, void *);
 /* obd_config.c */
 struct lustre_cfg *lustre_cfg_rename(struct lustre_cfg *cfg,
                                     const char *new_name);
@@ -139,6 +143,11 @@ struct obd_device *class_incref(struct obd_device *obd,
 void class_decref(struct obd_device *obd,
                   const char *scope, const void *source);
 void dump_exports(struct obd_device *obd, int locks);
+int class_config_llog_handler(const struct lu_env *env,
+                             struct llog_handle *handle,
+                             struct llog_rec_hdr *rec, void *data);
+int class_add_conn(struct obd_device *obd, struct lustre_cfg *lcfg);
+int class_add_uuid(const char *uuid, __u64 nid);
 
 /*obdecho*/
 #ifdef LPROCFS
@@ -162,6 +171,7 @@ struct config_llog_instance {
         void               *cfg_instance;
         struct super_block *cfg_sb;
         struct obd_uuid     cfg_uuid;
+       llog_cb_t           cfg_callback;
         int                 cfg_last_idx; /* for partial llog processing */
         int                 cfg_flags;
 };
@@ -2262,12 +2272,29 @@ static inline void fid2obdo(struct lu_fid *fid, struct obdo *oa)
         /* something here */
 }
 
+typedef int (*register_osp_cb)(void *data);
+
+struct osp_register_item {
+       struct obd_export **ori_exp;
+       register_osp_cb     ori_cb_func;
+       void               *ori_cb_data;
+       cfs_list_t          ori_list;
+       char                ori_name[MTI_NAME_MAXLEN];
+};
+
 /* I'm as embarrassed about this as you are.
  *
  * <shaver> // XXX do not look into _superhack with remaining eye
  * <shaver> // XXX if this were any uglier, I'd get my own show on MTV */
 extern int (*ptlrpc_put_connection_superhack)(struct ptlrpc_connection *c);
 
+/* obd_mount.c */
+int server_name2fsname(char *svname, char *fsname, char **endptr);
+int lustre_register_osp_item(char *ospname, struct obd_export **exp,
+                            register_osp_cb cb_func, void *cb_data);
+void lustre_deregister_osp_item(struct obd_export **exp);
+int tgt_name2ospname(char *ost_name, char *osp_name);
+
 /* sysctl.c */
 extern void obd_sysctl_init (void);
 extern void obd_sysctl_clean (void);
index d14a6dc..288003c 100644 (file)
@@ -266,7 +266,9 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
 
         /* In a more perfect world, we would hang a ptlrpc_client off of
          * obd_type and just use the values from there. */
-        if (!strcmp(name, LUSTRE_OSC_NAME)) {
+       if (!strcmp(name, LUSTRE_OSC_NAME) ||
+           (!strcmp(name, LUSTRE_OSP_NAME) &&
+            !is_osp_on_ost(lustre_cfg_buf(lcfg, 0)))) {
                 rq_portal = OST_REQUEST_PORTAL;
                 rp_portal = OSC_REPLY_PORTAL;
                 connect_op = OST_CONNECT;
@@ -274,7 +276,9 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
                 cli->cl_sp_to = LUSTRE_SP_OST;
                 ns_type = LDLM_NS_TYPE_OSC;
 
-        } else if (!strcmp(name, LUSTRE_MDC_NAME)) {
+       } else if (!strcmp(name, LUSTRE_MDC_NAME) ||
+                  (!strcmp(name, LUSTRE_OSP_NAME) &&
+                   is_osp_on_ost(lustre_cfg_buf(lcfg, 0)))) {
                 rq_portal = MDS_REQUEST_PORTAL;
                 rp_portal = MDC_REPLY_PORTAL;
                 connect_op = MDS_CONNECT;
index 7305317..b30165a 100644 (file)
@@ -997,7 +997,7 @@ int ll_fill_super(struct super_block *sb, struct vfsmount *mnt)
            Use the address of the super itself.*/
         cfg->cfg_instance = sb;
         cfg->cfg_uuid = lsi->lsi_llsbi->ll_sb_uuid;
-
+       cfg->cfg_callback = class_config_llog_handler;
         /* set up client obds */
         err = lustre_process_log(sb, profilenm, cfg);
         if (err < 0) {
index b1553ba..34d19fb 100644 (file)
@@ -214,6 +214,8 @@ struct config_llog_data *do_config_log_add(struct obd_device *obd,
         strcpy(cld->cld_logname, logname);
         if (cfg)
                 cld->cld_cfg = *cfg;
+       else
+               cld->cld_cfg.cfg_callback = class_config_llog_handler;
         cfs_mutex_init(&cld->cld_lock);
         cld->cld_cfg.cfg_last_idx = 0;
         cld->cld_cfg.cfg_flags = 0;
@@ -262,6 +264,10 @@ static struct config_llog_data *config_recover_log_add(struct obd_device *obd,
        if (IS_OST(lsi))
                 return NULL;
 
+       /* for osp-on-ost, see lustre_start_osp() */
+       if (IS_MDT(lsi) && lcfg.cfg_instance)
+               return NULL;
+
         /* we have to use different llog for clients and mdts for cmd
          * where only clients are notified if one of cmd server restarts */
         LASSERT(strlen(fsname) < sizeof(logname) / 2);
index 20e14dc..0d53c7e 100644 (file)
@@ -145,6 +145,7 @@ int class_add_uuid(const char *uuid, __u64 nid)
         }
         return 0;
 }
+EXPORT_SYMBOL(class_add_uuid);
 
 /* Delete the nids for one uuid if specified, otherwise delete all */
 int class_del_uuid(const char *uuid)
index f2344e2..02c4669 100644 (file)
@@ -800,6 +800,7 @@ int class_add_conn(struct obd_device *obd, struct lustre_cfg *lcfg)
 
         RETURN(rc);
 }
+EXPORT_SYMBOL(class_add_conn);
 
 /** Remove a failover nid location.
  */
@@ -1334,9 +1335,9 @@ extern int lustre_check_exclusion(struct super_block *sb, char *svname);
  * records, change uuids, etc), then class_process_config() resulting
  * net records.
  */
-static int class_config_llog_handler(const struct lu_env *env,
-                                    struct llog_handle *handle,
-                                    struct llog_rec_hdr *rec, void *data)
+int class_config_llog_handler(const struct lu_env *env,
+                             struct llog_handle *handle,
+                             struct llog_rec_hdr *rec, void *data)
 {
         struct config_llog_instance *clli = data;
         int cfg_len = rec->lrh_len;
@@ -1547,14 +1548,15 @@ out:
         }
         RETURN(rc);
 }
+EXPORT_SYMBOL(class_config_llog_handler);
 
 int class_config_parse_llog(const struct lu_env *env, struct llog_ctxt *ctxt,
                            char *name, struct config_llog_instance *cfg)
 {
        struct llog_process_cat_data     cd = {0, 0};
        struct llog_handle              *llh;
+       llog_cb_t                        callback;
        int                              rc;
-
        ENTRY;
 
        CDEBUG(D_INFO, "looking up llog %s\n", name);
@@ -1567,11 +1569,17 @@ int class_config_parse_llog(const struct lu_env *env, struct llog_ctxt *ctxt,
                GOTO(parse_out, rc);
 
        /* continue processing from where we last stopped to end-of-log */
-       if (cfg)
+       if (cfg) {
                cd.lpcd_first_idx = cfg->cfg_last_idx;
+               callback = cfg->cfg_callback;
+               LASSERT(callback != NULL);
+       } else {
+               callback = class_config_llog_handler;
+       }
+
        cd.lpcd_last_idx = 0;
 
-       rc = llog_process(env, llh, class_config_llog_handler, cfg, &cd);
+       rc = llog_process(env, llh, callback, cfg, &cd);
 
        CDEBUG(D_CONFIG, "Processed log %s gen %d-%d (rc=%d)\n", name,
               cd.lpcd_first_idx + 1, cd.lpcd_last_idx, rc);
index 2ca91e9..4c82860 100644 (file)
@@ -822,6 +822,609 @@ static int server_mgc_clear_fs(struct obd_device *mgc)
         RETURN(rc);
 }
 
+/** Get the fsname ("lustre") from the server name ("lustre-OST003F").
+ * @param [in] svname server name including type and index
+ * @param [out] fsname Buffer to copy filesystem name prefix into.
+ *  Must have at least 'strlen(fsname) + 1' chars.
+ * @param [out] endptr if endptr isn't NULL it is set to end of fsname
+ * rc < 0  on error
+ */
+int server_name2fsname(char *svname, char *fsname, char **endptr)
+{
+       char *dash = strrchr(svname, '-');
+       if (!dash) {
+               dash = strrchr(svname, ':');
+               if (!dash)
+                       return -EINVAL;
+       }
+
+       /* interpret <fsname>-MDTXXXXX-mdc as mdt, the better way is to pass
+        * in the fsname, then determine the server index */
+       if (!strcmp(LUSTRE_MDC_NAME, dash + 1)) {
+               dash--;
+               for (; dash > svname && *dash != '-' && *dash != ':'; dash--)
+                       ;
+               if (dash == svname)
+                       return -EINVAL;
+       }
+
+       if (fsname != NULL) {
+               strncpy(fsname, svname, dash - svname);
+               fsname[dash - svname] = '\0';
+       }
+
+       if (endptr != NULL)
+               *endptr = dash;
+
+       return 0;
+}
+EXPORT_SYMBOL(server_name2fsname);
+
+static int is_mdc_device(char *devname)
+{
+       char *ptr;
+       ptr = strrchr(devname, '-');
+       if (ptr != NULL && strcmp(ptr, "-mdc") == 0)
+               return 1;
+       return 0;
+}
+
+static int inline tgt_is_mdt0(char *tgtname)
+{
+       __u32 idx;
+       int   type;
+
+       type = server_name2index(tgtname, &idx, NULL);
+       if (type != LDD_F_SV_TYPE_MDT)
+               return 0;
+
+       return (idx == 0) ? 1 :0;
+}
+
+static int inline is_mdc_for_mdt0(char *devname)
+{
+       char   *ptr;
+
+       if (!is_mdc_device(devname))
+               return 0;
+
+       ptr = strrchr(devname, '-');
+       if (ptr == NULL)
+               return 0;
+
+       *ptr = 0;
+       if (tgt_is_mdt0(devname)) {
+               *ptr = '-';
+               return 1;
+       }
+       *ptr = '-';
+       return 0;
+}
+
+/**
+ * Convert OST/MDT name(fsname-OSTxxxx) to an osp name
+ * (fsname-MDT0000-osp-OSTxxxx), which will be used to
+ * communicate with MDT0 for this target.
+ **/
+int tgt_name2ospname(char *svname, char *ospname)
+{
+       char *fsname;
+       char *tgt;
+       int   rc;
+       ENTRY;
+
+       OBD_ALLOC(fsname, MTI_NAME_MAXLEN);
+       if (fsname == NULL)
+               RETURN(-ENOMEM);
+
+       rc = server_name2fsname(svname, fsname, &tgt);
+       if (rc != 0) {
+               CERROR("%s change fsname error: rc %d\n", svname, rc);
+               GOTO(cleanup, rc);
+       }
+
+       if (*tgt != '-' && *tgt != ':') {
+               CERROR("%s wrong svname name!\n", svname);
+               GOTO(cleanup, rc = -EINVAL);
+       }
+
+       tgt++;
+       if (strncmp(tgt, "OST", 3) != 0 && strncmp(tgt, "MDT", 3) != 0) {
+               CERROR("%s is not an OST or MDT target!\n", svname);
+               GOTO(cleanup, rc = -EINVAL);
+       }
+       sprintf(ospname, "%s-MDT0000-%s-%s", fsname, LUSTRE_OSP_NAME, tgt);
+cleanup:
+       if (fsname != NULL)
+               OBD_FREE(fsname, MTI_NAME_MAXLEN);
+       RETURN(rc);
+}
+EXPORT_SYMBOL(tgt_name2ospname);
+
+static CFS_LIST_HEAD(osp_register_list);
+CFS_DEFINE_MUTEX(osp_register_list_lock);
+
+int lustre_register_osp_item(char *ospname, struct obd_export **exp,
+                            register_osp_cb cb_func, void *cb_data)
+{
+       struct obd_device        *osp;
+       struct osp_register_item *ori;
+       ENTRY;
+
+       LASSERTF(strlen(ospname) < MTI_NAME_MAXLEN, "ospname is too long %s\n",
+                ospname);
+       LASSERT(exp != NULL && *exp == NULL);
+
+       OBD_ALLOC_PTR(ori);
+       if (ori == NULL)
+               RETURN(-ENOMEM);
+
+       cfs_mutex_lock(&osp_register_list_lock);
+
+       osp = class_name2obd(ospname);
+       if (osp != NULL && osp->obd_set_up == 1) {
+               struct obd_uuid *uuid;
+
+               OBD_ALLOC_PTR(uuid);
+               if (uuid == NULL) {
+                       cfs_mutex_unlock(&osp_register_list_lock);
+                       RETURN(-ENOMEM);
+               }
+               memcpy(uuid->uuid, ospname, strlen(ospname));
+               *exp = cfs_hash_lookup(osp->obd_uuid_hash, uuid);
+               OBD_FREE_PTR(uuid);
+       }
+
+       memcpy(ori->ori_name, ospname, strlen(ospname));
+       ori->ori_exp = exp;
+       ori->ori_cb_func = cb_func;
+       ori->ori_cb_data = cb_data;
+       CFS_INIT_LIST_HEAD(&ori->ori_list);
+       cfs_list_add(&ori->ori_list, &osp_register_list);
+
+       if (*exp != NULL && cb_func != NULL)
+               cb_func(cb_data);
+
+       cfs_mutex_unlock(&osp_register_list_lock);
+       RETURN(0);
+}
+EXPORT_SYMBOL(lustre_register_osp_item);
+
+void lustre_deregister_osp_item(struct obd_export **exp)
+{
+       struct osp_register_item *ori, *tmp;
+
+       cfs_mutex_lock(&osp_register_list_lock);
+       cfs_list_for_each_entry_safe(ori, tmp, &osp_register_list, ori_list) {
+               if (exp == ori->ori_exp) {
+                       if (*exp)
+                               class_export_put(*exp);
+                       cfs_list_del(&ori->ori_list);
+                       OBD_FREE_PTR(ori);
+                       break;
+               }
+       }
+       cfs_mutex_unlock(&osp_register_list_lock);
+}
+EXPORT_SYMBOL(lustre_deregister_osp_item);
+
+static void lustre_notify_osp_list(struct obd_export *exp)
+{
+       struct osp_register_item *ori, *tmp;
+       LASSERT(exp != NULL);
+
+       cfs_mutex_lock(&osp_register_list_lock);
+       cfs_list_for_each_entry_safe(ori, tmp, &osp_register_list, ori_list) {
+               if (strcmp(exp->exp_obd->obd_name, ori->ori_name))
+                       continue;
+               if (*ori->ori_exp != NULL)
+                       continue;
+               *ori->ori_exp = class_export_get(exp);
+               if (ori->ori_cb_func != NULL)
+                       ori->ori_cb_func(ori->ori_cb_data);
+       }
+       cfs_mutex_unlock(&osp_register_list_lock);
+}
+
+static int lustre_osp_connect(struct obd_device *osp)
+{
+       struct lu_env            env;
+       struct lu_context        session_ctx;
+       struct obd_export       *exp;
+       struct obd_uuid         *uuid = NULL;
+       struct obd_connect_data *data = NULL;
+       int                      rc;
+       ENTRY;
+
+       /* log has been fully processed, let clients connect */
+       rc = lu_env_init(&env, osp->obd_lu_dev->ld_type->ldt_ctx_tags);
+       if (rc != 0)
+               RETURN(rc);
+
+       lu_context_init(&session_ctx, LCT_SESSION);
+       session_ctx.lc_thread = NULL;
+       lu_context_enter(&session_ctx);
+       env.le_ses = &session_ctx;
+
+       OBD_ALLOC_PTR(data);
+       if (data == NULL)
+               GOTO(out, rc = -ENOMEM);
+
+       data->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_INDEX;
+       data->ocd_version = LUSTRE_VERSION_CODE;
+       data->ocd_ibits_known = MDS_INODELOCK_UPDATE;
+       data->ocd_connect_flags |= OBD_CONNECT_ACL | OBD_CONNECT_IBITS |
+                                  OBD_CONNECT_MDS_MDS | OBD_CONNECT_FID |
+                                  OBD_CONNECT_AT | OBD_CONNECT_FULL20 |
+                                  OBD_CONNECT_LIGHTWEIGHT;
+       OBD_ALLOC_PTR(uuid);
+       if (uuid == NULL)
+               GOTO(out, rc = -ENOMEM);
+
+       if (strlen(osp->obd_name) > sizeof(uuid->uuid)) {
+               CERROR("%s: Too long osp name %s, max_size is %d\n",
+                      osp->obd_name, osp->obd_name, (int)sizeof(uuid->uuid));
+               GOTO(out, rc = -EINVAL);
+       }
+       /* Use osp name as the uuid, so we find the export by
+        * osp name later */
+       memcpy(uuid->uuid, osp->obd_name, strlen(osp->obd_name));
+       rc = obd_connect(&env, &exp, osp, uuid, data, NULL);
+       if (rc != 0)
+               CERROR("%s: connect failed: rc = %d\n", osp->obd_name, rc);
+       else
+               lustre_notify_osp_list(exp);
+
+out:
+       if (data != NULL)
+               OBD_FREE_PTR(data);
+       if (uuid != NULL)
+               OBD_FREE_PTR(uuid);
+
+       lu_env_fini(&env);
+       lu_context_exit(&session_ctx);
+       lu_context_fini(&session_ctx);
+
+       RETURN(rc);
+}
+
+/**
+ * osp-on-ost is used by slaves (Non-MDT0 targets) to manage the connection
+ * to MDT0.
+ *
+ * The OSTs will communicate with MDT0 by the connection established by the
+ * osp-on-ost to get quota and fid sequence.
+ *
+ **/
+static int lustre_osp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi)
+{
+       struct obd_connect_data *data = NULL;
+       struct obd_device       *obd;
+       char                    *ospname = NULL;
+       char                    *ospuuid = NULL;
+       int                      rc;
+       ENTRY;
+
+       rc = class_add_uuid(lustre_cfg_string(lcfg, 1),
+                           lcfg->lcfg_nid);
+       if (rc) {
+               CERROR("%s: Can't add uuid: rc =%d\n", lsi->lsi_svname, rc);
+               GOTO(out, rc);
+       }
+
+       OBD_ALLOC(ospname, MTI_NAME_MAXLEN);
+       if (ospname == NULL)
+               GOTO(out, rc = -ENOMEM);
+
+       rc = tgt_name2ospname(lsi->lsi_svname, ospname);
+       if (rc != 0) {
+               CERROR("%s change ospname error: rc %d\n",
+                      lsi->lsi_svname, rc);
+               GOTO(out, rc);
+       }
+
+       OBD_ALLOC(ospuuid, MTI_NAME_MAXLEN);
+       if (ospuuid == NULL)
+               GOTO(out, rc = -ENOMEM);
+
+       sprintf(ospuuid, "%s_UUID", ospname);
+       rc = lustre_start_simple(ospname, LUSTRE_OSP_NAME,
+                                ospuuid, lustre_cfg_string(lcfg, 1),
+                                0, 0, 0);
+       if (rc) {
+               CERROR("%s: setup up failed: rc %d\n", ospname, rc);
+               GOTO(out, rc);
+       }
+
+       obd = class_name2obd(ospname);
+       LASSERT(obd != NULL);
+
+       rc = lustre_osp_connect(obd);
+       if (rc != 0)
+               CERROR("%s: connect failed: rc = %d\n", ospname, rc);
+out:
+       if (data != NULL)
+               OBD_FREE_PTR(data);
+       if (ospname != NULL)
+               OBD_FREE(ospname, MTI_NAME_MAXLEN);
+       if (ospuuid != NULL)
+               OBD_FREE(ospuuid, MTI_NAME_MAXLEN);
+
+       RETURN(rc);
+}
+
+static int lustre_osp_add_conn(struct lustre_cfg *cfg,
+                               struct lustre_sb_info *lsi)
+{
+       struct lustre_cfg_bufs *bufs = NULL;
+       struct lustre_cfg      *lcfg = NULL;
+       char                   *ospname = NULL;
+       struct obd_device      *osp;
+       int                     rc;
+       ENTRY;
+
+       OBD_ALLOC(ospname, MTI_NAME_MAXLEN);
+       if (ospname == NULL)
+               GOTO(out, rc = -ENOMEM);
+
+       rc = tgt_name2ospname(lsi->lsi_svname, ospname);
+       if (rc != 0) {
+               CERROR("%s change ospname error: rc %d\n",
+                      lsi->lsi_svname, rc);
+               GOTO(out, rc);
+       }
+
+       OBD_ALLOC_PTR(bufs);
+       if (bufs == NULL)
+               RETURN(-ENOMEM);
+
+       lustre_cfg_bufs_reset(bufs, ospname);
+       lustre_cfg_bufs_set_string(bufs, 1,
+                                  lustre_cfg_string(cfg, 1));
+
+       lcfg = lustre_cfg_new(LCFG_ADD_CONN, bufs);
+
+       osp = class_name2obd(ospname);
+       if (osp == NULL) {
+               CERROR("Can not find %s\n", ospname);
+               GOTO(out, rc = -EINVAL);
+       }
+
+       rc = class_add_conn(osp, lcfg);
+       if (rc)
+               CERROR("%s: can't add conn: rc = %d\n", ospname, rc);
+
+out:
+       if (bufs != NULL)
+               OBD_FREE_PTR(bufs);
+       if (lcfg != NULL)
+               lustre_cfg_free(lcfg);
+       if (ospname != NULL)
+               OBD_FREE(ospname, MTI_NAME_MAXLEN);
+
+       RETURN(rc);
+}
+
+/**
+ * Retrieve MDT nids from the client log, then start the osp-on-ost device.
+ * there are only two scenarios which would include mdt nid.
+ * 1.
+ * marker   5 (flags=0x01, v2.1.54.0) lustre-MDT0000  'add mdc' xxx-
+ * add_uuid  nid=192.168.122.162@tcp(0x20000c0a87aa2)  0:  1:192.168.122.162@tcp
+ * attach    0:lustre-MDT0000-mdc  1:mdc  2:lustre-clilmv_UUID
+ * setup     0:lustre-MDT0000-mdc  1:lustre-MDT0000_UUID  2:192.168.122.162@tcp
+ * add_uuid  nid=192.168.172.1@tcp(0x20000c0a8ac01)  0:  1:192.168.172.1@tcp
+ * add_conn  0:lustre-MDT0000-mdc  1:192.168.172.1@tcp
+ * modify_mdc_tgts add 0:lustre-clilmv  1:lustre-MDT0000_UUID xxxx
+ * marker   5 (flags=0x02, v2.1.54.0) lustre-MDT0000  'add mdc' xxxx-
+ * 2.
+ * marker   7 (flags=0x01, v2.1.54.0) lustre-MDT0000  'add failnid' xxxx-
+ * add_uuid  nid=192.168.122.2@tcp(0x20000c0a87a02)  0:  1:192.168.122.2@tcp
+ * add_conn  0:lustre-MDT0000-mdc  1:192.168.122.2@tcp
+ * marker   7 (flags=0x02, v2.1.54.0) lustre-MDT0000  'add failnid' xxxx-
+**/
+static int client_osp_config_process(const struct lu_env *env,
+                                    struct llog_handle *handle,
+                                    struct llog_rec_hdr *rec, void *data)
+{
+       struct config_llog_instance *clli = data;
+       int                          cfg_len = rec->lrh_len;
+       char                        *cfg_buf = (char *) (rec + 1);
+       struct lustre_cfg           *lcfg = NULL;
+       struct lustre_sb_info       *lsi;
+       int                          rc = 0, swab = 0;
+       ENTRY;
+
+       if (rec->lrh_type != OBD_CFG_REC) {
+               CERROR("Unknown llog record type %#x encountered\n",
+                      rec->lrh_type);
+               RETURN(-EINVAL);
+       }
+
+       LASSERT(clli->cfg_sb != NULL);
+       lsi = s2lsi(clli->cfg_sb);
+
+       lcfg = (struct lustre_cfg *)cfg_buf;
+       if (lcfg->lcfg_version == __swab32(LUSTRE_CFG_VERSION)) {
+               lustre_swab_lustre_cfg(lcfg);
+               swab = 1;
+       }
+
+       rc = lustre_cfg_sanity_check(cfg_buf, cfg_len);
+       if (rc)
+               GOTO(out, rc);
+
+       switch (lcfg->lcfg_command) {
+       case LCFG_MARKER: {
+               struct cfg_marker *marker = lustre_cfg_buf(lcfg, 1);
+
+               lustre_swab_cfg_marker(marker, swab,
+                                      LUSTRE_CFG_BUFLEN(lcfg, 1));
+               if (marker->cm_flags & CM_SKIP ||
+                   marker->cm_flags & CM_EXCLUDE)
+                       GOTO(out, rc = 0);
+
+               if (!tgt_is_mdt0(marker->cm_tgtname))
+                       GOTO(out, rc = 0);
+
+               if(!strncmp(marker->cm_comment, "add mdc", 7) ||
+                  !strncmp(marker->cm_comment, "add failnid", 11)) {
+                       if (marker->cm_flags & CM_START) {
+                               clli->cfg_flags = CFG_F_MARKER;
+                               /* This hack is to differentiate the
+                                * ADD_UUID is come from "add mdc" record
+                                * or from "add failnid" record. */
+                               if (!strncmp(marker->cm_comment,
+                                            "add failnid", 11))
+                                       clli->cfg_flags |= CFG_F_SKIP;
+                       } else if (marker->cm_flags & CM_END) {
+                               clli->cfg_flags = 0;
+                       }
+               }
+               break;
+       }
+       case LCFG_ADD_UUID: {
+               if (clli->cfg_flags == CFG_F_MARKER) {
+                       rc = lustre_osp_setup(lcfg, lsi);
+               } else if (clli->cfg_flags == (CFG_F_MARKER | CFG_F_SKIP)) {
+                       rc = class_add_uuid(lustre_cfg_string(lcfg, 1),
+                                       lcfg->lcfg_nid);
+                       if (rc)
+                               CERROR("%s: Fail to add uuid, rc:%d\n",
+                                      lsi->lsi_svname, rc);
+               }
+               break;
+       }
+       case LCFG_ADD_CONN: {
+               if (is_mdc_for_mdt0(lustre_cfg_string(lcfg, 0)))
+                       rc = lustre_osp_add_conn(lcfg, lsi);
+               break;
+       }
+       default:
+               break;
+       }
+out:
+       RETURN(rc);
+}
+
+static int lustre_disconnect_osp(struct super_block *sb)
+{
+       struct lustre_sb_info           *lsi = s2lsi(sb);
+       struct obd_device               *osp;
+       char                            *ospname = NULL;
+       char                            *logname = NULL;
+       struct lustre_cfg               *lcfg = NULL;
+       struct lustre_cfg_bufs          *bufs = NULL;
+       struct config_llog_instance     *cfg = NULL;
+       int                              rc;
+       ENTRY;
+
+       LASSERT(IS_OST(lsi) || IS_MDT(lsi));
+       OBD_ALLOC(logname, MTI_NAME_MAXLEN);
+       if (logname == NULL)
+               RETURN(-ENOMEM);
+
+       OBD_ALLOC(ospname, MTI_NAME_MAXLEN);
+       if (ospname == NULL)
+               GOTO(out, rc = -ENOMEM);
+
+       rc = server_name2fsname(lsi->lsi_svname, ospname, NULL);
+       if (rc != 0) {
+               CERROR("%s: get fsname error: %d\n",
+                      lsi->lsi_svname, rc);
+               GOTO(out, rc);
+       }
+       sprintf(logname, "%s-client", ospname);
+
+       OBD_ALLOC_PTR(cfg);
+       if (cfg == NULL)
+               GOTO(out, rc = -ENOMEM);
+
+       /* end log first */
+       cfg->cfg_instance = sb;
+       rc = lustre_end_log(sb, logname, cfg);
+       if (rc != 0) {
+               CERROR("Can't end config log %s\n", ospname);
+               GOTO(out, rc);
+       }
+
+       rc = tgt_name2ospname(lsi->lsi_svname, ospname);
+       if (rc != 0) {
+               CERROR("%s: get osp name error: %d\n",
+                      lsi->lsi_svname, rc);
+               GOTO(out, rc);
+       }
+
+       osp = class_name2obd(ospname);
+        if (osp == NULL) {
+                CERROR("Can't find osp-on-ost %s\n", ospname);
+                GOTO(out, rc = -ENOENT);
+        }
+
+       OBD_ALLOC_PTR(bufs);
+       if (bufs == NULL)
+               GOTO(out, rc = -ENOMEM);
+
+       lustre_cfg_bufs_reset(bufs, osp->obd_name);
+       lustre_cfg_bufs_set_string(bufs, 1, NULL);
+       lcfg = lustre_cfg_new(LCFG_CLEANUP, bufs);
+       if (!lcfg)
+               GOTO(out, rc = -ENOMEM);
+
+       /* Disconnect import first. NULL is passed for the '@env', since
+        * it will not be used for the 'osp-on-ost'. (see osp_shutdown()) */
+       rc = osp->obd_lu_dev->ld_ops->ldo_process_config(NULL, osp->obd_lu_dev,
+                                                        lcfg);
+out:
+       if (lcfg)
+               lustre_cfg_free(lcfg);
+       if (bufs)
+               OBD_FREE_PTR(bufs);
+       if (cfg)
+               OBD_FREE_PTR(cfg);
+       if (ospname)
+               OBD_FREE(ospname, MTI_NAME_MAXLEN);
+       if (logname)
+               OBD_FREE(logname, MTI_NAME_MAXLEN);
+       RETURN(rc);
+}
+
+/**
+ * Stop the osp(fsname-MDT0000-osp-OSTxxxx) for an OST target.
+ **/
+static int lustre_stop_osp(struct super_block *sb)
+{
+       struct lustre_sb_info   *lsi = s2lsi(sb);
+       struct obd_device       *osp = NULL;
+       char                    *ospname = NULL;
+       int                      rc = 0;
+       ENTRY;
+
+       LASSERT(IS_OST(lsi) || IS_MDT(lsi));
+       OBD_ALLOC(ospname, MTI_NAME_MAXLEN);
+
+       rc = tgt_name2ospname(lsi->lsi_svname, ospname);
+       if (rc != 0) {
+               CERROR("%s get fsname error: rc %d\n",
+                      lsi->lsi_svname, rc);
+               GOTO(cleanup, rc);
+       }
+
+       osp = class_name2obd(ospname);
+       if (osp == NULL) {
+               CERROR("Can not find osp-on-ost %s\n", ospname);
+               GOTO(cleanup, rc = -ENOENT);
+       }
+
+       osp->obd_force = 1;
+       rc = class_manual_cleanup(osp);
+
+cleanup:
+       if (ospname != NULL)
+               OBD_FREE(ospname, MTI_NAME_MAXLEN);
+       RETURN(rc);
+}
+
 CFS_DEFINE_MUTEX(server_start_lock);
 
 /* Stop MDS/OSS if nobody is using them */
@@ -872,34 +1475,6 @@ int server_mti_print(char *title, struct mgs_target_info *mti)
         return(0);
 }
 
-/** Get the fsname ("lustre") from the server name ("lustre-OST003F").
- * @param [in] svname server name including type and index
- * @param [out] fsname Buffer to copy filesystem name prefix into.
- *  Must have at least 'strlen(fsname) + 1' chars.
- * @param [out] endptr if endptr isn't NULL it is set to end of fsname
- * rc < 0  on error
- */
-static int server_name2fsname(char *svname, char *fsname, char **endptr)
-{
-       char *p;
-
-       p = strstr(svname, "-OST");
-       if (p == NULL)
-               p = strstr(svname, "-MDT");
-       if (p == NULL)
-               return -1;
-
-       if (fsname) {
-               strncpy(fsname, svname, p - svname);
-               fsname[p - svname] = '\0';
-       }
-
-       if (endptr != NULL)
-               *endptr = p;
-
-       return 0;
-}
-
 /**
  * Get service name (svname) from string
  * rc < 0 on error
@@ -1129,6 +1704,66 @@ out:
 
 }
 
+/**
+ * Start the osp(fsname-MDT0000-osp-OSTxxxx) for an OST target,
+ * which would be used to communicate with MDT0 for quota and FID.
+ **/
+static int lustre_start_osp(struct super_block *sb)
+{
+       struct lustre_sb_info       *lsi = s2lsi(sb);
+       struct config_llog_instance *cfg = NULL;
+       struct obd_device           *osp;
+       char                        *ospname = NULL;
+       char                        *logname = NULL;
+       char                        *tgt;
+       int                          rc;
+       ENTRY;
+
+       LASSERT(IS_OST(lsi) || IS_MDT(lsi));
+       OBD_ALLOC(ospname, MTI_NAME_MAXLEN);
+       OBD_ALLOC(logname, MTI_NAME_MAXLEN);
+       if (ospname == NULL || logname == NULL)
+               GOTO(cleanup, rc = -ENOMEM);
+
+       rc = server_name2fsname(lsi->lsi_svname, ospname, &tgt);
+       if (rc != 0) {
+               CERROR("%s change fsname error: rc %d\n",
+                      lsi->lsi_svname, rc);
+               GOTO(cleanup, rc);
+       }
+       sprintf(logname, "%s-client", ospname);
+
+       rc = tgt_name2ospname(lsi->lsi_svname, ospname);
+       if (rc != 0) {
+               CERROR("%s change ospname error: rc %d\n",
+                      lsi->lsi_svname, rc);
+               GOTO(cleanup, rc);
+       }
+
+       osp = class_name2obd(ospname);
+       if (osp != NULL)
+               GOTO(cleanup, rc = 0);
+
+       OBD_ALLOC_PTR(cfg);
+       if (cfg == NULL)
+               GOTO(cleanup, rc = -ENOMEM);
+
+       cfg->cfg_callback = client_osp_config_process;
+       cfg->cfg_instance = sb;
+
+       rc = lustre_process_log(sb, logname, cfg);
+
+cleanup:
+       if (ospname != NULL)
+               OBD_FREE(ospname, MTI_NAME_MAXLEN);
+       if (logname != NULL)
+               OBD_FREE(logname, MTI_NAME_MAXLEN);
+       if (cfg != NULL)
+               OBD_FREE_PTR(cfg);
+
+       RETURN(rc);
+}
+
 /** Start server targets: MDTs and OSTs
  */
 static int server_start_targets(struct super_block *sb, struct vfsmount *mnt)
@@ -1205,6 +1840,7 @@ static int server_start_targets(struct super_block *sb, struct vfsmount *mnt)
 
        /* Start targets using the llog named for the target */
        memset(&cfg, 0, sizeof(cfg));
+       cfg.cfg_callback = class_config_llog_handler;
        rc = lustre_process_log(sb, lsi->lsi_svname, &cfg);
        if (rc) {
                CERROR("failed to start server %s: %d\n",
@@ -1215,55 +1851,61 @@ static int server_start_targets(struct super_block *sb, struct vfsmount *mnt)
                GOTO(out_mgc, rc);
        }
 
-out_mgc:
-        /* Release the mgc fs for others to use */
-       if (lsi->lsi_srv_mnt)
-               server_mgc_clear_fs(lsi->lsi_mgc);
+       obd = class_name2obd(lsi->lsi_svname);
+       if (!obd) {
+               CERROR("no server named %s was started\n", lsi->lsi_svname);
+               GOTO(out_mgc, rc = -ENXIO);
+       }
 
-        if (!rc) {
-               obd = class_name2obd(lsi->lsi_svname);
-                if (!obd) {
-                        CERROR("no server named %s was started\n",
-                              lsi->lsi_svname);
-                        RETURN(-ENXIO);
-                }
+       if (IS_OST(lsi) || IS_MDT(lsi)) {
+               rc = lustre_start_osp(sb);
+               if (rc) {
+                       CERROR("%s: failed to start OSP: %d\n",
+                              lsi->lsi_svname, rc);
+                       GOTO(out_mgc, rc);
+               }
+       }
 
-                server_notify_target(sb, obd);
+       server_notify_target(sb, obd);
 
-                /* calculate recovery timeout, do it after lustre_process_log */
-                server_calc_timeout(lsi, obd);
+       /* calculate recovery timeout, do it after lustre_process_log */
+       server_calc_timeout(lsi, obd);
 
-                /* log has been fully processed */
-                obd_notify(obd, NULL, OBD_NOTIFY_CONFIG, (void *)CONFIG_LOG);
+       /* log has been fully processed */
+       obd_notify(obd, NULL, OBD_NOTIFY_CONFIG, (void *)CONFIG_LOG);
 
-               /* log has been fully processed, let clients connect */
-               dev = obd->obd_lu_dev;
-               if (dev && dev->ld_ops->ldo_prepare) {
-                       rc = lu_env_init(&env, dev->ld_type->ldt_ctx_tags);
-                       if (rc == 0) {
-                               struct lu_context  session_ctx;
+       /* log has been fully processed, let clients connect */
+       dev = obd->obd_lu_dev;
+       if (dev && dev->ld_ops->ldo_prepare) {
+               rc = lu_env_init(&env, dev->ld_type->ldt_ctx_tags);
+               if (rc == 0) {
+                       struct lu_context  session_ctx;
 
-                               lu_context_init(&session_ctx, LCT_SESSION);
-                               session_ctx.lc_thread = NULL;
-                               lu_context_enter(&session_ctx);
-                               env.le_ses = &session_ctx;
+                       lu_context_init(&session_ctx, LCT_SESSION);
+                       session_ctx.lc_thread = NULL;
+                       lu_context_enter(&session_ctx);
+                       env.le_ses = &session_ctx;
 
-                               dev->ld_ops->ldo_prepare(&env, NULL, dev);
+                       dev->ld_ops->ldo_prepare(&env, NULL, dev);
 
-                               lu_env_fini(&env);
-                               lu_context_exit(&session_ctx);
-                               lu_context_fini(&session_ctx);
-                       }
+                       lu_env_fini(&env);
+                       lu_context_exit(&session_ctx);
+                       lu_context_fini(&session_ctx);
                }
+       }
 
-               /* abort recovery only on the complete stack:
-                * many devices can be involved */
-               if ((lsi->lsi_lmd->lmd_flags & LMD_FLG_ABORT_RECOV) &&
-                   (OBP(obd, iocontrol))) {
-                       obd_iocontrol(OBD_IOC_ABORT_RECOVERY,
-                                     obd->obd_self_export, 0, NULL, NULL);
-               }
-        }
+       /* abort recovery only on the complete stack:
+        * many devices can be involved */
+       if ((lsi->lsi_lmd->lmd_flags & LMD_FLG_ABORT_RECOV) &&
+           (OBP(obd, iocontrol))) {
+               obd_iocontrol(OBD_IOC_ABORT_RECOVERY, obd->obd_self_export, 0,
+                             NULL, NULL);
+       }
+
+out_mgc:
+        /* Release the mgc fs for others to use */
+       if (lsi->lsi_srv_mnt)
+               server_mgc_clear_fs(lsi->lsi_mgc);
 
         RETURN(rc);
 }
@@ -1459,6 +2101,12 @@ static void server_put_super(struct super_block *sb)
        if (IS_MDT(lsi) && (lsi->lsi_lmd->lmd_flags & LMD_FLG_NOSVC))
                 snprintf(tmpname, tmpname_sz, "MGS");
 
+       /* disconnect the osp-on-ost first to drain off the inflight request */
+       if (IS_OST(lsi) || IS_MDT(lsi)) {
+               if (lustre_disconnect_osp(sb) < 0)
+                       CERROR("%s: Fail to disconnect osp-on-ost!\n", tmpname);
+       }
+
         /* Stop the target */
         if (!(lsi->lsi_lmd->lmd_flags & LMD_FLG_NOSVC) &&
            (IS_MDT(lsi) || IS_OST(lsi))) {
@@ -1499,6 +2147,11 @@ static void server_put_super(struct super_block *sb)
                         server_stop_mgs(sb);
         }
 
+       if (IS_OST(lsi) || IS_MDT(lsi)) {
+               if (lustre_stop_osp(sb) < 0)
+                       CERROR("%s: Fail to stop osp-on-ost!\n", tmpname);
+       }
+
         /* Clean the mgc and sb */
         lustre_common_put_super(sb);
 
index 788d5a4..e504d0f 100644 (file)
@@ -1,5 +1,6 @@
 MODULES := osp
 osp-objs := osp_dev.o osp_object.o osp_precreate.o osp_sync.o lproc_osp.o
+osp-objs += osp_ost.o
 
 EXTRA_DIST = $(osp-objs:.o=.c) osp_internal.h
 
index 9c533a9..cf9ee4f 100644 (file)
@@ -183,24 +183,23 @@ out:
 
 static void osp_last_used_fini(const struct lu_env *env, struct osp_device *d)
 {
-       lu_object_put(env, &d->opd_last_used_file->do_lu);
-       d->opd_last_used_file = NULL;
+       if (d->opd_last_used_file != NULL) {
+               lu_object_put(env, &d->opd_last_used_file->do_lu);
+               d->opd_last_used_file = NULL;
+       }
 }
 
-static int osp_shutdown(const struct lu_env *env, struct osp_device *d)
+int osp_disconnect(struct osp_device *d)
 {
-       struct obd_import       *imp;
-       int                      rc = 0;
-       ENTRY;
-
-       /* release last_used file */
-       osp_last_used_fini(env, d);
+       struct obd_import *imp;
+       int rc = 0;
 
        imp = d->opd_obd->u.cli.cl_import;
 
        /* Mark import deactivated now, so we don't try to reconnect if any
         * of the cleanup RPCs fails (e.g. ldlm cancel, etc).  We don't
         * fully deactivate the import, or that would drop all requests. */
+       LASSERT(imp != NULL);
        cfs_spin_lock(&imp->imp_lock);
        imp->imp_deactive = 1;
        cfs_spin_unlock(&imp->imp_lock);
@@ -219,6 +218,25 @@ static int osp_shutdown(const struct lu_env *env, struct osp_device *d)
 
        ptlrpc_invalidate_import(imp);
 
+       RETURN(rc);
+}
+
+static int osp_shutdown(const struct lu_env *env, struct osp_device *d)
+{
+       int                      rc = 0;
+       ENTRY;
+
+       if (is_osp_on_ost(d->opd_obd->obd_name)) {
+               rc = osp_disconnect(d);
+               RETURN(rc);
+       }
+
+       LASSERT(env);
+       /* release last_used file */
+       osp_last_used_fini(env, d);
+
+       rc = osp_disconnect(d);
+
        /* stop precreate thread */
        osp_precreate_fini(d);
 
@@ -239,7 +257,8 @@ static int osp_process_config(const struct lu_env *env,
 
        switch (lcfg->lcfg_command) {
        case LCFG_CLEANUP:
-               lu_dev_del_linkage(dev->ld_site, dev);
+               if (!is_osp_on_ost(d->opd_obd->obd_name))
+                       lu_dev_del_linkage(dev->ld_site, dev);
                rc = osp_shutdown(env, d);
                break;
        case LCFG_PARAM:
@@ -343,7 +362,7 @@ static int osp_sync(const struct lu_env *env, struct dt_device *dev)
        RETURN(0);
 }
 
-static const struct dt_device_operations osp_dt_ops = {
+const struct dt_device_operations osp_dt_ops = {
        .dt_statfs      = osp_statfs,
        .dt_sync        = osp_sync,
 };
@@ -585,7 +604,10 @@ static struct lu_device *osp_device_alloc(const struct lu_env *env,
 
                l = osp2lu_dev(m);
                dt_device_init(&m->opd_dt_dev, t);
-               rc = osp_init0(env, m, t, lcfg);
+               if (is_osp_on_ost(lustre_cfg_string(lcfg, 0)))
+                       rc = osp_init_for_ost(env, m, t, lcfg);
+               else
+                       rc = osp_init0(env, m, t, lcfg);
                if (rc != 0) {
                        osp_device_free(env, l);
                        l = ERR_PTR(rc);
@@ -603,8 +625,11 @@ static struct lu_device *osp_device_fini(const struct lu_env *env,
 
        ENTRY;
 
-       LASSERT(m->opd_storage_exp);
-       obd_disconnect(m->opd_storage_exp);
+       if (m->opd_storage_exp)
+               obd_disconnect(m->opd_storage_exp);
+
+       if (is_osp_on_ost(m->opd_obd->obd_name))
+               osp_fini_for_ost(m);
 
        imp = m->opd_obd->u.cli.cl_import;
 
@@ -661,6 +686,8 @@ static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp,
                RETURN(rc);
 
        *exp = class_conn2export(&conn);
+       if (is_osp_on_ost(obd->obd_name))
+               osp->opd_exp = *exp;
 
        /* Why should there ever be more than 1 connect? */
        osp->opd_connects++;
@@ -680,7 +707,8 @@ static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp,
                                 OBD_CONNECT_OSS_CAPA |
                                 OBD_CONNECT_REQPORTAL |
                                 OBD_CONNECT_SKIP_ORPHAN |
-                                OBD_CONNECT_VERSION;
+                                OBD_CONNECT_VERSION |
+                                OBD_CONNECT_FID;
        ocd->ocd_version = LUSTRE_VERSION_CODE;
        LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX);
        ocd->ocd_index = data->ocd_index;
@@ -707,7 +735,6 @@ static int osp_obd_disconnect(struct obd_export *exp)
        struct obd_device *obd = exp->exp_obd;
        struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
        int                rc;
-
        ENTRY;
 
        /* Only disconnect the underlying layers on the final disconnect. */
@@ -715,9 +742,14 @@ static int osp_obd_disconnect(struct obd_export *exp)
        osp->opd_connects--;
 
        rc = class_disconnect(exp);
+       if (rc) {
+               CERROR("%s: class disconnect error: rc = %d\n",
+                      obd->obd_name, rc);
+               RETURN(rc);
+       }
 
        /* destroy the device */
-       if (rc == 0)
+       if (!is_osp_on_ost(obd->obd_name))
                class_manual_cleanup(obd);
 
        RETURN(rc);
@@ -798,12 +830,16 @@ static int osp_import_event(struct obd_device *obd, struct obd_import *imp,
        case IMP_EVENT_DISCON:
                d->opd_got_disconnected = 1;
                d->opd_imp_connected = 0;
+               if (is_osp_on_ost(d->opd_obd->obd_name))
+                       break;
                osp_pre_update_status(d, -ENODEV);
                cfs_waitq_signal(&d->opd_pre_waitq);
                CDEBUG(D_HA, "got disconnected\n");
                break;
        case IMP_EVENT_INACTIVE:
                d->opd_imp_active = 0;
+               if (is_osp_on_ost(d->opd_obd->obd_name))
+                       break;
                osp_pre_update_status(d, -ENODEV);
                cfs_waitq_signal(&d->opd_pre_waitq);
                CDEBUG(D_HA, "got inactive\n");
@@ -814,10 +850,19 @@ static int osp_import_event(struct obd_device *obd, struct obd_import *imp,
                        d->opd_new_connection = 1;
                d->opd_imp_connected = 1;
                d->opd_imp_seen_connected = 1;
+               if (is_osp_on_ost(d->opd_obd->obd_name))
+                       break;
                cfs_waitq_signal(&d->opd_pre_waitq);
                __osp_sync_check_for_work(d);
                CDEBUG(D_HA, "got connected\n");
                break;
+       case IMP_EVENT_INVALIDATE:
+               if (obd->obd_namespace == NULL)
+                       break;
+               ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
+               break;
+       case IMP_EVENT_OCD:
+               break;
        default:
                CERROR("%s: unsupported import event: %#x\n",
                       obd->obd_name, event);
index 7a82bca..5f4b77c 100644 (file)
@@ -42,6 +42,7 @@
 #define _OSP_INTERNAL_H
 
 #include <obd.h>
+#include <obd_class.h>
 #include <dt_object.h>
 #include <lustre_fid.h>
 
@@ -182,6 +183,7 @@ struct osp_object {
 };
 
 extern struct lu_object_operations osp_lu_obj_ops;
+extern const struct dt_device_operations osp_dt_ops;
 
 struct osp_thread_info {
        struct lu_buf            osi_lb;
@@ -314,4 +316,10 @@ int osp_sync_init(const struct lu_env *env, struct osp_device *d);
 int osp_sync_fini(struct osp_device *d);
 void __osp_sync_check_for_work(struct osp_device *d);
 
+/* osp_ost.c */
+int osp_init_for_ost(const struct lu_env *env, struct osp_device *m,
+                    struct lu_device_type *ldt, struct lustre_cfg *cfg);
+int osp_disconnect(struct osp_device *d);
+int osp_fini_for_ost(struct osp_device *osp);
+
 #endif
diff --git a/lustre/osp/osp_ost.c b/lustre/osp/osp_ost.c
new file mode 100644 (file)
index 0000000..2fb6a55
--- /dev/null
@@ -0,0 +1,175 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2012 Intel, Inc.
+ * Use is subject to license terms.
+ *
+ * lustre/osp/osp_ost.c
+ *
+ * OSP on OST for communicating with MDT0
+ *
+ * Author: <di.wang@whamcloud.com>
+ */
+#ifndef EXPORT_SYMTAB
+# define EXPORT_SYMTAB
+#endif
+#define DEBUG_SUBSYSTEM S_OST
+
+#include <obd_class.h>
+#include <lustre_param.h>
+#include <lustre_log.h>
+
+#include "osp_internal.h"
+
+static int osp_name2fsname(char *ospname, char *fsname)
+{
+       char *ptr;
+
+       LASSERT(ospname != NULL);
+       LASSERT(fsname != NULL);
+       if (!is_osp_on_ost(ospname))
+               return -EINVAL;
+
+       sprintf(fsname, "-%s-", LUSTRE_OSP_NAME);
+
+       ptr = strstr(ospname, fsname);
+       if (ptr) {
+               strncpy(fsname, ospname, ptr - ospname);
+               fsname[ptr - ospname] = '\0';
+       }
+       return 0;
+}
+
+static int osp_setup_for_ost(const struct lu_env *env, struct osp_device *osp,
+                             char *nidstring)
+{
+       struct lustre_cfg_bufs  *bufs = NULL;
+       struct lustre_cfg       *lcfg = NULL;
+       char                    *ospname = osp->opd_obd->obd_name;
+       char                    *fsname = NULL;
+       char                    *server_uuid = NULL;
+       class_uuid_t             uuid;
+       struct obd_import       *imp;
+       int                      rc;
+       ENTRY;
+
+       OBD_ALLOC_PTR(bufs);
+       if (bufs == NULL)
+               RETURN(-ENOMEM);
+
+       OBD_ALLOC(fsname, strlen(ospname));
+       if (fsname == NULL)
+               GOTO(out, rc = -ENOMEM);
+
+       rc = osp_name2fsname(ospname, fsname);
+       if (rc) {
+               CERROR("%s: name change error: rc %d\n", ospname, rc);
+               GOTO(out, rc);
+       }
+
+       OBD_ALLOC(server_uuid, strlen(fsname) + 15);
+       if (server_uuid == NULL)
+               GOTO(out, rc = -ENOMEM);
+
+       sprintf(server_uuid, "%s-MDT0000_UUID", fsname);
+       lustre_cfg_bufs_reset(bufs, ospname);
+       lustre_cfg_bufs_set_string(bufs, 1, server_uuid);
+       lustre_cfg_bufs_set_string(bufs, 2, nidstring);
+       lcfg = lustre_cfg_new(LCFG_SETUP, bufs);
+       if (lcfg == NULL)
+               GOTO(out, rc = -ENOMEM);
+
+       rc = client_obd_setup(osp->opd_obd, lcfg);
+       if (rc != 0) {
+               CERROR("%s: client obd setup error: rc = %d\n",
+                      osp->opd_obd->obd_name, rc);
+               GOTO(out, rc);
+       }
+
+       imp = osp->opd_obd->u.cli.cl_import;
+       rc = ptlrpc_init_import(imp);
+       if (rc)
+               GOTO(out, rc);
+
+       ll_generate_random_uuid(uuid);
+       class_uuid_unparse(uuid, &osp->opd_cluuid);
+out:
+       if (bufs != NULL)
+               OBD_FREE_PTR(bufs);
+       if (server_uuid != NULL)
+               OBD_FREE(server_uuid, strlen(fsname) + 15);
+       if (fsname != NULL)
+               OBD_FREE(fsname, strlen(ospname));
+       if (lcfg != NULL)
+               lustre_cfg_free(lcfg);
+       if (rc)
+               client_obd_cleanup(osp->opd_obd);
+       RETURN(rc);
+}
+
+int osp_fini_for_ost(struct osp_device *osp)
+{
+       if (osp->opd_exp != NULL)
+               class_disconnect(osp->opd_exp);
+       return 0;
+}
+
+int osp_init_for_ost(const struct lu_env *env, struct osp_device *osp,
+                    struct lu_device_type *ldt, struct lustre_cfg *cfg)
+{
+       struct lprocfs_static_vars lvars = { 0 };
+       int                        rc;
+       ENTRY;
+
+       osp->opd_obd = class_name2obd(lustre_cfg_string(cfg, 0));
+       if (osp->opd_obd == NULL) {
+               CERROR("Cannot find obd with name %s\n",
+                      lustre_cfg_string(cfg, 0));
+               RETURN(-ENODEV);
+       }
+
+       osp->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops;
+       osp->opd_dt_dev.dd_ops = &osp_dt_ops;
+       osp->opd_obd->obd_lu_dev = &osp->opd_dt_dev.dd_lu_dev;
+
+       rc = ptlrpcd_addref();
+       if (rc) {
+               CERROR("%s: ptlrpcd addref error: rc =%d\n",
+                      osp->opd_obd->obd_name, rc);
+               RETURN(rc);
+       }
+
+       rc = osp_setup_for_ost(env, osp, lustre_cfg_string(cfg, 1));
+       if (rc) {
+               CERROR("%s: osp_setup_for_ost error: rc =%d\n",
+                      osp->opd_obd->obd_name, rc);
+               ptlrpcd_decref();
+               RETURN(rc);
+       }
+
+       lprocfs_osp_init_vars(&lvars);
+       if (lprocfs_obd_setup(osp->opd_obd, lvars.obd_vars) == 0)
+               ptlrpc_lprocfs_register_obd(osp->opd_obd);
+
+       RETURN(0);
+}
index f013315..9afb3ea 100644 (file)
@@ -480,6 +480,7 @@ load_modules_local() {
                else
                        load_module obdfilter/obdfilter
                fi
+               load_module osp/osp
     fi