Branch: HEAD

some fixes for WB cache
1)reorganization for cobd_setup, add cache_real_exp, master_real_exp for real
  connection
2)the fix in lmc for wrong lov_index

diff --git a/lustre/cmobd/cm_obd.c b/lustre/cmobd/cm_obd.c
index 43fa1e3..3492fe1 100644 (file)
--- a/lustre/cmobd/cm_obd.c
+++ b/lustre/cmobd/cm_obd.c
@@ -154,7 +154,8 @@ static int cmobd_setup(struct obd_device *obd, obd_count len, void *buf)
 
         /* connecting master */
         memset(&conn, 0, sizeof(conn));
-        rc = obd_connect(&conn, cmobd->master_obd, &obd->obd_uuid, NULL, 0);
+        rc = obd_connect(&conn, cmobd->master_obd, &obd->obd_uuid, 
+                         NULL, OBD_OPT_REAL_CLIENT);
         if (rc)
                 RETURN(rc);
         cmobd->master_exp = class_conn2export(&conn);

diff --git a/lustre/cobd/cache_obd.c b/lustre/cobd/cache_obd.c
index 67d69a6..bca2a8c 100644 (file)
--- a/lustre/cobd/cache_obd.c
+++ b/lustre/cobd/cache_obd.c
@@ -96,7 +96,7 @@ static int cobd_setup(struct obd_device *obd, obd_count len, void *buf)
 
         /* connecting master */
         memset(&conn, 0, sizeof(conn));
-        rc = obd_connect(&conn, master_obd, &obd->obd_uuid, NULL, 0);
+        rc = class_connect(&conn, master_obd, &obd->obd_uuid);
         if (rc)
                GOTO(put_names, rc);
         
@@ -105,7 +105,7 @@ static int cobd_setup(struct obd_device *obd, obd_count len, void *buf)
         /* getting master obd */
         cache_obd = class_name2obd(cobd->cache_name);
         if (!cache_obd) {
-                obd_disconnect(cobd->master_exp, 0);
+                class_disconnect(cobd->master_exp, 0);
                 CERROR("can't find cache obd by name %s\n",
                        cobd->cache_name);
                 GOTO(put_names, rc);
@@ -113,12 +113,14 @@ static int cobd_setup(struct obd_device *obd, obd_count len, void *buf)
 
         /* connecting master */
         memset(&conn, 0, sizeof(conn));
-        rc = obd_connect(&conn, cache_obd, &obd->obd_uuid, NULL, 0);
+        rc = class_connect(&conn, cache_obd, &obd->obd_uuid);
         if (rc) {
-                obd_disconnect(cobd->master_exp, 0);
+                class_disconnect(cobd->master_exp, 0);
                 GOTO(put_names, rc);
         }
         cobd->cache_exp = class_conn2export(&conn);
+        /*default set cache on*/
+        cobd->cache_on = 1;
         EXIT;
 put_names:
         if (rc) {
@@ -138,6 +140,7 @@ put_names:
 static int cobd_cleanup(struct obd_device *obd, int flags)
 {
         struct cache_obd  *cobd = &obd->u.cobd;
+        int rc = 0;
         ENTRY;
 
         if (!list_empty(&obd->obd_exports))
@@ -149,7 +152,18 @@ static int cobd_cleanup(struct obd_device *obd, int flags)
         if (cobd->master_name)
                 OBD_FREE(cobd->master_name, 
                          strlen(cobd->master_name) + 1);
- 
+        
+        rc = class_disconnect(cobd->master_exp, flags);
+        if (rc) {
+                CERROR("error disconnecting master, err %d\n",
+                       rc);
+        }
+        rc = class_disconnect(cobd->cache_exp, flags);
+        if (rc) {
+                CERROR("error disconnecting master, err %d\n",
+                       rc);
+        }
+
         RETURN(0);
 }
 
@@ -157,12 +171,19 @@ static inline struct obd_export *
 cobd_get_exp(struct obd_device *obd)
 {
         struct cache_obd *cobd = &obd->u.cobd;
-        if (cobd->cache_on)  
+        if (cobd->cache_on) {
+                CDEBUG(D_TRACE, "get cache exp %p \n", cobd->cache_exp); 
+                if (cobd->cache_real_exp)
+                       return cobd->cache_real_exp;
                 return cobd->cache_exp;
+        }
+        CDEBUG(D_TRACE, "get master exp %p \n", cobd->master_exp);
+        if (cobd->master_real_exp)
+                return cobd->master_real_exp; 
         return cobd->master_exp;
 }
 
-static int client_obd_connect(struct obd_device *obd, char *name,
+static int client_obd_connect(struct obd_device *obd, struct obd_export *exp,
                               struct lustre_handle *conn,
                               struct obd_connect_data *data,
                               unsigned long flags)
@@ -172,20 +193,16 @@ static int client_obd_connect(struct obd_device *obd, char *name,
         ENTRY;
  
         LASSERT(obd);
-        LASSERT(name);
         LASSERT(conn);
         
-        cli_obd = class_name2obd(name);
-        if (cli_obd == NULL) {
-                CERROR("%s: unable to find a client for obd: %s\n",
-                       obd->obd_name, name);
+        cli_obd = class_exp2obd(exp);
+        if (cli_obd == NULL) 
                 RETURN(-EINVAL);
-        }
+
         rc = obd_connect(conn, cli_obd, &obd->obd_uuid, data, flags);
-        if (rc) {
-                CERROR("error connecting to %s, err %d\n",
-                       name, rc);
-        }
+        if (rc) 
+                CERROR("error connecting err %d\n", rc);
+        
         RETURN(rc);
 }
 
@@ -214,12 +231,34 @@ cobd_connect(struct lustre_handle *conn, struct obd_device *obd,
              struct obd_uuid *cluuid, struct obd_connect_data *data,
              unsigned long flags)
 {
-        struct obd_export *cobd_exp;
+        struct lustre_handle cache_conn = { 0 };
+        struct cache_obd *cobd = &obd->u.cobd;
+        struct obd_export *exp, *cobd_exp;
+        int rc = 0;
+        ENTRY;
+
+        rc = class_connect(conn, obd, cluuid);
+        if (rc)
+                RETURN(rc);
+        exp = class_conn2export(conn);
 
         cobd_exp = cobd_get_exp(obd);
         
-        return obd_connect(conn, class_exp2obd(cobd_exp), cluuid, 
-                           data, flags); 
+        /* connecting cache */
+        rc = client_obd_connect(obd, cobd_exp, &cache_conn, 
+                                data, flags);
+        if (rc)
+                GOTO(err_discon, rc);
+       
+        cobd->cache_real_exp = class_conn2export(&cache_conn);
+        cobd->cache_on = 1;
+        EXIT;
+err_discon:
+        if (rc)
+                class_disconnect(exp, 0);
+        else
+                class_export_put(exp);
+        RETURN(rc);
 }
 
 static int
@@ -239,8 +278,8 @@ cobd_disconnect(struct obd_export *exp, unsigned long flags)
         }
         cobd_exp = cobd_get_exp(obd);
         
-        rc = obd_disconnect(cobd_exp, flags);
-      
+        rc = client_obd_disconnect(obd, cobd_exp, flags);
+
         class_disconnect(exp, flags);
         
         RETURN(rc);
@@ -742,12 +781,12 @@ static int cobd_iocontrol(unsigned int cmd, struct obd_export *exp,
                 if (!cobd->cache_on) {
                         struct lustre_handle conn = {0};
 
-                        rc = client_obd_disconnect(obd, cobd->master_exp, 0);
-                        rc = client_obd_connect(obd, cobd->cache_name, &conn,
+                        rc = client_obd_disconnect(obd, cobd->master_real_exp, 0);
+                        rc = client_obd_connect(obd, cobd->cache_exp, &conn,
                                                 NULL, 0);
                         if (rc)
                                 GOTO(out, rc);
-                        cobd->cache_exp = class_conn2export(&conn);
+                        cobd->cache_real_exp = class_conn2export(&conn);
                         cobd->cache_on = 1;
                 }
                 break;
@@ -762,12 +801,12 @@ static int cobd_iocontrol(unsigned int cmd, struct obd_export *exp,
                         easize = cache->u.cli.cl_max_mds_easize; 
                         cooksize = cache->u.cli.cl_max_mds_cookiesize;
                         
-                        rc = client_obd_disconnect(obd, cobd->cache_exp, 0);
-                        rc = client_obd_connect(obd, cobd->master_name, &conn,
+                        rc = client_obd_disconnect(obd, cobd->cache_real_exp, 0);
+                        rc = client_obd_connect(obd, cobd->master_exp, &conn,
                                                 NULL, 0);
                         if (rc)
                                 GOTO(out, rc);
-                        cobd->master_exp = class_conn2export(&conn);
+                        cobd->master_real_exp = class_conn2export(&conn);
 
                         master = class_exp2obd(cobd->master_exp);
                         master->u.cli.cl_max_mds_easize = easize;

diff --git a/lustre/include/linux/obd.h b/lustre/include/linux/obd.h
index 260b94b..f264325 100644 (file)
--- a/lustre/include/linux/obd.h
+++ b/lustre/include/linux/obd.h
@@ -455,6 +455,8 @@ struct echo_client_obd {
 struct cache_obd {
         struct obd_export      *master_exp; /* local connection to master obd */
         struct obd_export      *cache_exp;  /* local connection to cache obd */
+        struct obd_export      *cache_real_exp;
+        struct obd_export      *master_real_exp;
         struct obd_device      *master;
         struct obd_device      *cache;
         char                   *master_name;

diff --git a/lustre/tests/cmobd.sh b/lustre/tests/cmobd.sh
index c36f83e..109c588 100644 (file)
--- a/lustre/tests/cmobd.sh
+++ b/lustre/tests/cmobd.sh
@@ -6,10 +6,13 @@ config=${1:-cmobd.xml}
 LMC=${LMC:-lmc}
 TMP=${TMP:-/tmp}
 
-CMOBD_MDS1=${CMOBD_MDS1:-"cmobd-mds1"}
+COBD_MDS=${COBD_MDS:-"cobd_mds"}
+COBD_OST=${COBD_OST:-"cobd_ost"}
+CMOBD_MDS=${CMOBD_MDS:-"cmobd-mds"}
 CMOBD_MDS2=${CMOBD_MDS2:-"cmobd-mds2"}
 
 MASTER_LMV=${MASTER_LMV1:-master-lmv1}
+CACHE_LMV=${MASTER_LMV1:-cache-lmv1}
 
 CACHE_MDS1=${CACHE_MDS1:-"cache-mds1"}
 CACHE_MDS2=${CACHE_MDS2:-"cache-mds2"}
@@ -27,8 +30,11 @@ OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-"kml"}
 MDSSIZE=${MDSSIZE:-100000}
 
 MASTER_LOV=${MASTER_LOV:-"master-lov1"}
+CACHE_LOV=${CACHE_LOV:-"cache-lov1"}
 MASTER_OST=${MASTER_OST:-"master-ost1"}
+CACHE_OST=${CACHE_OST:-"cache-ost1"}
 OST_MASTER_DEV=$TMP/ost1-master-localhost
+OST_CACHE_DEV=$TMP/ost1-cache-localhost
 
 OSTSIZE=${OSTSIZE:-100000}
 
@@ -41,12 +47,10 @@ BACK_FSTYPE=${BACK_FSTYPE:-ldiskfs}
 
 NETTYPE=${NETTYPE:-tcp}
 NIDTYPE=${NIDTYPE:-$NETTYPE}
+STRIPE_SIZE=${STRIPE_SIZE:-65536}
 
 NODE=${NODE:-"localhost"}
 
-CLIENTS=${CLIENTS:-1}
-MODE=${MODE:-lmv}
-
 rm -f $config
 
 h2tcp () {
@@ -59,64 +63,46 @@ h2tcp () {
 ${LMC} -m $config --add net --node $NODE --nid `h2$NIDTYPE $NODE` \
 --nettype $NETTYPE || exit 1
 
-${LMC} -m $config --add mds --node $NODE --mds $CACHE_MDS1 --fstype $FSTYPE \
---backfstype $BACK_FSTYPE --dev $MDS1_CACHE_DEV  \
---mountfsoptions $MDS_MOUNT_OPTS --size $MDSSIZE --format || exit 10
-
-if test "x$CLIENTS" = "x2"; then
-        ${LMC} -m $config --add mds --node $NODE --mds $CACHE_MDS2 \
-        --fstype $FSTYPE --backfstype $BACK_FSTYPE --dev $MDS2_CACHE_DEV \
-        --mountfsoptions $MDS_MOUNT_OPTS --size $MDSSIZE --format || exit 10
-fi
-
-if test "x$MODE" = "xmds"; then
-        ${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS1 \
-        --fstype $BACK_FSTYPE --dev $MDS1_MASTER_DEV --size $MDSSIZE --format || exit 10
-else
-        ${LMC} -m $config --add lmv --lmv $MASTER_LMV || exit 12
-
-        ${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS1 \
-        --fstype $BACK_FSTYPE --dev $MDS1_MASTER_DEV --size $MDSSIZE \
-        --lmv $MASTER_LMV --format || exit 10
-
-        ${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS2 \
-        --fstype $BACK_FSTYPE --dev $MDS2_MASTER_DEV --size $MDSSIZE \
-        --lmv $MASTER_LMV --format || exit 10
-fi
-
-if test "x$MODE" = "xmds"; then
-        ${LMC} -m $config --add lov --lov $MASTER_LOV --mds $MASTER_MDS1 \
-        --stripe_sz 65536 --stripe_cnt $STRIPECNT --stripe_pattern 0 || exit 20
-else
-        ${LMC} -m $config --add lov --lov $MASTER_LOV --lmv $MASTER_LMV \
-        --stripe_sz 65536 --stripe_cnt $STRIPECNT --stripe_pattern 0 || exit 20
-fi
+
+#add cache lmv for cache lmv 
+${LMC} -m $config --add lmv --lmv $CACHE_LMV || exit 12
+
+${LMC} -m $config --add mds --node $NODE --mds $CACHE_MDS1 \
+--fstype $FSTYPE --backfstype $BACK_FSTYPE --dev $MDS1_CACHE_DEV \
+--mountfsoptions $MDS_MOUNT_OPTS --size $MDSSIZE --lmv $CACHE_LMV --format || exit 10
+
+${LMC} -m $config --add mds --node $NODE --mds $CACHE_MDS2 \
+--fstype $FSTYPE --backfstype $BACK_FSTYPE --dev $MDS2_CACHE_DEV \
+--mountfsoptions $MDS_MOUNT_OPTS --size $MDSSIZE --lmv $CACHE_LMV --format || exit 10
+
+#add master lmv for master lmv
+${LMC} -m $config --add lmv --lmv $MASTER_LMV || exit 12
+
+${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS1 \
+--fstype $BACK_FSTYPE --dev $MDS1_MASTER_DEV --size $MDSSIZE \
+--lmv $MASTER_LMV --format || exit 10
+
+${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS2 \
+--fstype $BACK_FSTYPE --dev $MDS2_MASTER_DEV --size $MDSSIZE \
+--lmv $MASTER_LMV --format || exit 10
+
+${LMC} -m $config --add lov --lov $MASTER_LOV --lmv $CACHE_LMV \
+--stripe_sz $STRIPE_SIZE --stripe_cnt $STRIPECNT --stripe_pattern 0 || exit 20
+
+${LMC} -m $config --add lov --lov $CACHE_LOV --lmv $MASTER_LMV \
+--stripe_sz $STRIPE_SIZE --stripe_cnt $STRIPECNT --stripe_pattern 0 || exit 20
 
 ${LMC} -m $config --add ost --ost $MASTER_OST --node $NODE --lov $MASTER_LOV \
 --fstype $BACK_FSTYPE --dev $OST_MASTER_DEV --size $OSTSIZE  || exit 21
 
-if test "x$MODE" = "xmds"; then
-        ${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_MDS1 \
-        --master_obd $MASTER_MDS1 --cache_obd $CACHE_MDS1 || exit 23 
-else
-        ${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_MDS1 \
-        --master_obd $MASTER_LMV --cache_obd $CACHE_MDS1 || exit 23 
-fi
-
-if test "x$CLIENTS" = "x2"; then
-        if test "x$MODE" = "xmds"; then
-                ${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_MDS2 \
-                --master_obd $MASTER_MDS1 --cache_obd $CACHE_MDS2 || exit 23 
-        else
-                ${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_MDS2 \
-                --master_obd $MASTER_LMV --cache_obd $CACHE_MDS2 || exit 23 
-        fi
-fi
+${LMC} -m $config --add ost --ost $CACHE_OST --node $NODE --lov $CACHE_LOV \
+--fstype $BACK_FSTYPE --dev $OST_CACHE_DEV --size $OSTSIZE  || exit 21
 
-${LMC} -m $config --add mtpt --node $NODE --path /mnt/lustre \
---mds $CACHE_MDS1 --lov $MASTER_LOV || exit 30
+${LMC} -m $config --add cobd --node $NODE --cobd $COBD_MDS \
+--master_obd $MASTER_LMV --cache_obd $CACHE_LMV || exit 22
+
+${LMC} -m $config --add cobd --node $NODE --cobd $COBD_OST \
+--master_obd $MASTER_LOV --cache_obd $CACHE_LOV || exit 22
 
-if test "x$CLIENTS" = "x2"; then
-        ${LMC} -m $config --add mtpt --node $NODE --path /mnt/lustre1 \
-        --mds $CACHE_MDS2 --lov $MASTER_LOV || exit 30
-fi
+${LMC} -m $config --add mtpt --node $NODE --path /mnt/lustre \
+--lmv $COBD_MDS --lov $COBD_OST || exit 30

diff --git a/lustre/utils/lmc b/lustre/utils/lmc
index 44d6648..35b78be 100755 (executable)
--- a/lustre/utils/lmc
+++ b/lustre/utils/lmc
@@ -496,9 +496,10 @@ class GenConfig:
         lov.setAttribute("stripepattern", str(pattern))
         return lov
 
-    def lov_tgt(self, obd_uuid, index, generation):
+    def lov_tgt(self, obd_uuid, lov_uuid, index, generation):
         tgt = self.doc.createElement('lov_tgt')
         tgt.setAttribute("uuidref", obd_uuid)
+        tgt.setAttribute("lov_uuid", lov_uuid)
         tgt.setAttribute("index", index)
         tgt.setAttribute("generation", generation)
         tgt.setAttribute("active", '1')
@@ -730,6 +731,7 @@ def lov_mod_obd(gen, lustre, lov, tgt, osc_uuid, options):
 
 def lov_add_obd(gen, lustre, lov, osc_uuid, options):
     lov_name = getName(lov)
+    lov_uuid = getUUID(lov)
     if options.index:
         lov_index = get_option_int(options, 'index')
         for tgt in lustre.getElementsByTagName('lov_tgt'):
@@ -745,16 +747,18 @@ def lov_add_obd(gen, lustre, lov, osc_uuid, options):
          for tgt in lustre.getElementsByTagName('lov_tgt'):
              uuidref = tgt.getAttribute('uuidref')
              tmp = int(tgt.getAttribute('index'))
+             own_lov_uuid = tgt.getAttribute('lov_uuid')
              if tmp != lov_index:
                  error('malformed xml: LOV targets are not ordered; found index '+str(tmp)+', expected '+str(lov_index)+'.')
              uuidref = tgt.getAttribute('uuidref')
              if uuidref == '':
                  lov_mod_obd(gen, lustre, lov, tgt, osc_uuid, options)
                  return
-             lov_index = lov_index + 1
+            if own_lov_uuid == lov_uuid:  
+               lov_index = lov_index + 1
 
-    lov.appendChild(gen.lov_tgt(osc_uuid, str(lov_index), '1'))
-    addUpdate(gen, lustre, gen.add(getUUID(lov), osc_uuid, str(lov_index), '1'))
+    lov.appendChild(gen.lov_tgt(osc_uuid, lov_uuid, str(lov_index), '1'))
+    addUpdate(gen, lustre, gen.add(getUUID(lov), lov_uuid, str(lov_index), '1'))
 
 def lov_del_obd(gen, lustre, lov, osc_uuid, options):
     lov_name = getName(lov)