From 234ce16dba60f2e2c2177e5cde21efd75285e4b4 Mon Sep 17 00:00:00 2001 From: Alex Zhuravlev Date: Sun, 30 Sep 2012 21:38:02 +0400 Subject: [PATCH] LU-1943 zfs-osd: different fixes for zfs - echo_create_md_object() to mark la_mode is valid - ofd_statfs() to set avail/free blocks to 2 when the counters are converted to 4K units. otherwise with zfs 2 blocks of size 128K can be > 0.01% leading to failures in sanity/27* - zfs-osd to initialize LMA - zfs-osd's osd_attr_get() to calculate size of directory from blocks and do not use cached size (as we don't control it as with regular files) - zfs-osd to track enabled/disabled status for SA and follow it - take from orion additional locking in osd_object_sa_dirty_add() and osd_object_sa_dirty_rele() - lu_cache_shrink() do not purge objects if __GFP_FS is not set this should prevent deadlock when used with zfs backend - ost_checksum_bulk() to allocate new page to simulate corruption, otherwise it corrupts zfs's internal buffers which do not rely on PageUptoDate flag - t-f to modprobe zfs module if osd-zfs is in use Signed-off-by: Alex Zhuravlev Change-Id: I027483162735cd0fb4be9fa8bd12c2f2dc43b44a Reviewed-on: http://review.whamcloud.com/4141 Tested-by: Hudson Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Mike Pershin --- lustre/obdclass/lu_object.c | 38 +++++++++++++++++++++++++++--- lustre/obdecho/echo_client.c | 2 +- lustre/ofd/ofd_obd.c | 8 +++---- lustre/osd-zfs/osd_handler.c | 21 +++++++++++++++++ lustre/osd-zfs/osd_internal.h | 27 +++++++++++++++++++++ lustre/osd-zfs/osd_object.c | 37 +++++++++++++++++++++++++++++ lustre/osd-zfs/osd_xattr.c | 11 ++++----- lustre/ost/ost_handler.c | 32 ++++++++++++++++++++----- lustre/tests/conf-sanity.sh | 53 ++++++++++++++++++++++++++++++++++++++++-- lustre/tests/sanity.sh | 10 ++++++++ lustre/tests/test-framework.sh | 1 + 11 files changed, 217 insertions(+), 23 deletions(-) diff --git a/lustre/obdclass/lu_object.c b/lustre/obdclass/lu_object.c index 9eaa1ae..1f9eb53 100644 --- a/lustre/obdclass/lu_object.c +++ b/lustre/obdclass/lu_object.c @@ -1797,6 +1797,24 @@ static void lu_site_stats_get(cfs_hash_t *hs, #ifdef __KERNEL__ +/* + * There exists a potential lock inversion deadlock scenario when using + * Lustre on top of ZFS. This occurs between one of ZFS's + * buf_hash_table.ht_lock's, and Lustre's lu_sites_guard lock. Essentially, + * thread A will take the lu_sites_guard lock and sleep on the ht_lock, + * while thread B will take the ht_lock and sleep on the lu_sites_guard + * lock. Obviously neither thread will wake and drop their respective hold + * on their lock. + * + * To prevent this from happening we must ensure the lu_sites_guard lock is + * not taken while down this code path. ZFS reliably does not set the + * __GFP_FS bit in its code paths, so this can be used to determine if it + * is safe to take the lu_sites_guard lock. + * + * Ideally we should accurately return the remaining number of cached + * objects without taking the lu_sites_guard lock, but this is not + * possible in the current implementation. + */ static int lu_cache_shrink(SHRINKER_ARGS(sc, nr_to_scan, gfp_mask)) { lu_site_stats_t stats; @@ -1806,12 +1824,26 @@ static int lu_cache_shrink(SHRINKER_ARGS(sc, nr_to_scan, gfp_mask)) int remain = shrink_param(sc, nr_to_scan); CFS_LIST_HEAD(splice); - if (remain != 0) { - if (!(shrink_param(sc, gfp_mask) & __GFP_FS)) + if (!(shrink_param(sc, gfp_mask) & __GFP_FS)) { + if (remain != 0) return -1; - CDEBUG(D_INODE, "Shrink %d objects\n", remain); + else + /* We must not take the lu_sites_guard lock when + * __GFP_FS is *not* set because of the deadlock + * possibility detailed above. Additionally, + * since we cannot determine the number of + * objects in the cache without taking this + * lock, we're in a particularly tough spot. As + * a result, we'll just lie and say our cache is + * empty. This _should_ be ok, as we can't + * reclaim objects when __GFP_FS is *not* set + * anyways. + */ + return 0; } + CDEBUG(D_INODE, "Shrink %d objects\n", remain); + cfs_mutex_lock(&lu_sites_guard); cfs_list_for_each_entry_safe(s, tmp, &lu_sites, ls_linkage) { if (shrink_param(sc, nr_to_scan) != 0) { diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c index 3ee8138..b51d57e 100644 --- a/lustre/obdecho/echo_client.c +++ b/lustre/obdecho/echo_client.c @@ -1606,7 +1606,7 @@ static int echo_create_md_object(const struct lu_env *env, } ma->ma_attr.la_mode = mode; - ma->ma_attr.la_valid = LA_CTIME; + ma->ma_attr.la_valid = LA_CTIME | LA_MODE; ma->ma_attr.la_ctime = cfs_time_current_64(); if (name != NULL) { diff --git a/lustre/ofd/ofd_obd.c b/lustre/ofd/ofd_obd.c index 4895d31..1f8e917 100644 --- a/lustre/ofd/ofd_obd.c +++ b/lustre/ofd/ofd_obd.c @@ -729,10 +729,6 @@ static int ofd_statfs(const struct lu_env *env, struct obd_export *exp, osfs->os_blocks, osfs->os_bfree, osfs->os_bavail, osfs->os_files, osfs->os_ffree, osfs->os_state); - if (OBD_FAIL_CHECK_VALUE(OBD_FAIL_OST_ENOSPC, - ofd->ofd_lut.lut_lsd.lsd_ost_index)) - osfs->os_bfree = osfs->os_bavail = 2; - if (OBD_FAIL_CHECK_VALUE(OBD_FAIL_OST_ENOINO, ofd->ofd_lut.lut_lsd.lsd_ost_index)) osfs->os_ffree = 0; @@ -753,6 +749,10 @@ static int ofd_statfs(const struct lu_env *env, struct obd_export *exp, osfs->os_bsize = 1 << COMPAT_BSIZE_SHIFT; } + if (OBD_FAIL_CHECK_VALUE(OBD_FAIL_OST_ENOSPC, + ofd->ofd_lut.lut_lsd.lsd_ost_index)) + osfs->os_bfree = osfs->os_bavail = 2; + EXIT; out: return rc; diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c index bba18f7..7b85f40 100644 --- a/lustre/osd-zfs/osd_handler.c +++ b/lustre/osd-zfs/osd_handler.c @@ -505,9 +505,17 @@ static int osd_shutdown(const struct lu_env *env, struct osd_device *o) RETURN(0); } +static void osd_xattr_changed_cb(void *arg, uint64_t newval) +{ + struct osd_device *osd = arg; + + osd->od_xattr_in_sa = (newval == ZFS_XATTR_SA); +} + static int osd_mount(const struct lu_env *env, struct osd_device *o, struct lustre_cfg *cfg) { + struct dsl_dataset *ds; char *dev = lustre_cfg_string(cfg, 1); dmu_buf_t *rootdb; int rc; @@ -529,6 +537,13 @@ static int osd_mount(const struct lu_env *env, RETURN(rc); } + ds = dmu_objset_ds(o->od_objset.os); + LASSERT(ds); + rc = dsl_prop_register(ds, "xattr", osd_xattr_changed_cb, o); + if (rc) + CERROR("%s: cat not register xattr callback, ignore: %d\n", + o->od_svname, rc); + rc = __osd_obj2dbuf(env, o->od_objset.os, o->od_objset.root, &rootdb, root_tag); if (rc) { @@ -668,6 +683,7 @@ static struct lu_device *osd_device_fini(const struct lu_env *env, struct lu_device *d) { struct osd_device *o = osd_dev(d); + struct dsl_dataset *ds; int rc; ENTRY; @@ -675,6 +691,11 @@ static struct lu_device *osd_device_fini(const struct lu_env *env, osd_oi_fini(env, o); if (o->od_objset.os) { + ds = dmu_objset_ds(o->od_objset.os); + rc = dsl_prop_unregister(ds, "xattr", osd_xattr_changed_cb, o); + if (rc) + CERROR("%s: dsl_prop_unregister xattr error %d\n", + o->od_svname, rc); arc_remove_prune_callback(o->arc_prune_cb); o->arc_prune_cb = NULL; osd_sync(env, lu2dt_dev(d)); diff --git a/lustre/osd-zfs/osd_internal.h b/lustre/osd-zfs/osd_internal.h index cb9b8a5..d3025b7 100644 --- a/lustre/osd-zfs/osd_internal.h +++ b/lustre/osd-zfs/osd_internal.h @@ -147,6 +147,7 @@ struct osd_thread_info { char oti_str[64]; char oti_key[MAXNAMELEN + 1]; + struct lustre_mdt_attrs oti_mdt_attrs; struct lu_attr oti_la; struct osa_attr oti_osa; @@ -215,6 +216,7 @@ struct osd_device { uint64_t od_ost_compat_grp0; unsigned int od_rdonly:1, + od_xattr_in_sa:1, od_quota_iused_est:1; char od_mntdev[128]; char od_svname[128]; @@ -409,6 +411,31 @@ int osd_xattr_del(const struct lu_env *env, struct dt_object *dt, struct lustre_capa *capa); int osd_xattr_list(const struct lu_env *env, struct dt_object *dt, struct lu_buf *lb, struct lustre_capa *capa); +void __osd_xattr_declare_set(const struct lu_env *env, struct osd_object *obj, + int vallen, const char *name, struct osd_thandle *oh); +int __osd_sa_xattr_set(const struct lu_env *env, struct osd_object *obj, + const struct lu_buf *buf, const char *name, int fl, + struct osd_thandle *oh);; +int __osd_xattr_set(const struct lu_env *env, struct osd_object *obj, + const struct lu_buf *buf, const char *name, int fl, + struct osd_thandle *oh); +static inline int +osd_xattr_set_internal(const struct lu_env *env, struct osd_object *obj, + const struct lu_buf *buf, const char *name, int fl, + struct osd_thandle *oh, struct lustre_capa *capa) +{ + int rc; + + if (osd_obj2dev(obj)->od_xattr_in_sa) { + rc = __osd_sa_xattr_set(env, obj, buf, name, fl, oh); + if (rc == -EFBIG) + rc = __osd_xattr_set(env, obj, buf, name, fl, oh); + } else { + rc = __osd_xattr_set(env, obj, buf, name, fl, oh); + } + + return rc; +} #endif #endif /* _OSD_INTERNAL_H */ diff --git a/lustre/osd-zfs/osd_object.c b/lustre/osd-zfs/osd_object.c index e88f4e1..bb10a1f 100644 --- a/lustre/osd-zfs/osd_object.c +++ b/lustre/osd-zfs/osd_object.c @@ -125,8 +125,10 @@ osd_object_sa_dirty_add(struct osd_object *obj, struct osd_thandle *oh) return; cfs_down(&oh->ot_sa_lock); + cfs_write_lock(&obj->oo_attr_lock); if (likely(cfs_list_empty(&obj->oo_sa_linkage))) cfs_list_add(&obj->oo_sa_linkage, &oh->ot_sa_list); + cfs_write_unlock(&obj->oo_attr_lock); cfs_up(&oh->ot_sa_lock); } @@ -142,7 +144,9 @@ void osd_object_sa_dirty_rele(struct osd_thandle *oh) obj = cfs_list_entry(oh->ot_sa_list.next, struct osd_object, oo_sa_linkage); sa_spill_rele(obj->oo_sa_hdl); + cfs_write_lock(&obj->oo_attr_lock); cfs_list_del_init(&obj->oo_sa_linkage); + cfs_write_unlock(&obj->oo_attr_lock); } cfs_up(&oh->ot_sa_lock); } @@ -731,6 +735,10 @@ static int osd_attr_get(const struct lu_env *env, * from within sa_object_size() can block on a mutex, so * we can't call sa_object_size() holding rwlock */ sa_object_size(obj->oo_sa_hdl, &blksize, &blocks); + /* we do not control size of indices, so always calculate + * it from number of blocks reported by DMU */ + if (S_ISDIR(attr->la_mode)) + attr->la_size = 512 * blocks; /* Block size may be not set; suggest maximal I/O transfers. */ if (blksize == 0) blksize = 1ULL << SPA_MAXBLOCKSHIFT; @@ -1079,6 +1087,9 @@ static int osd_declare_object_create(const struct lu_env *env, dmu_tx_hold_sa_create(oh->ot_tx, ZFS_SA_BASE_ATTR_SIZE); + __osd_xattr_declare_set(env, obj, sizeof(struct lustre_mdt_attrs), + XATTR_NAME_LMA, oh); + RETURN(osd_declare_quota(env, osd, attr->la_uid, attr->la_gid, 1, oh, false, NULL, false)); } @@ -1361,6 +1372,24 @@ static osd_obj_type_f osd_create_type_f(enum dt_format_type type) /* * Primitives for directory (i.e. ZAP) handling */ +static inline int osd_init_lma(const struct lu_env *env, struct osd_object *obj, + const struct lu_fid *fid, struct osd_thandle *oh) +{ + struct osd_thread_info *info = osd_oti_get(env); + struct lustre_mdt_attrs *lma = &info->oti_mdt_attrs; + struct lu_buf buf; + int rc; + + lustre_lma_init(lma, fid); + lustre_lma_swab(lma); + buf.lb_buf = lma; + buf.lb_len = sizeof(*lma); + + rc = osd_xattr_set_internal(env, obj, &buf, XATTR_NAME_LMA, + LU_XATTR_CREATE, oh, BYPASS_CAPA); + + return rc; +} /* * Concurrency: @dt is write locked. @@ -1436,6 +1465,14 @@ static int osd_object_create(const struct lu_env *env, struct dt_object *dt, LASSERT(ergo(rc == 0, dt_object_exists(dt))); LASSERT(osd_invariant(obj)); + rc = osd_init_lma(env, obj, fid, oh); + if (rc) { + CERROR("%s: can not set LMA on "DFID": rc = %d\n", + osd->od_svname, PFID(fid), rc); + /* ignore errors during LMA initialization */ + rc = 0; + } + out: cfs_up(&obj->oo_guard); RETURN(rc); diff --git a/lustre/osd-zfs/osd_xattr.c b/lustre/osd-zfs/osd_xattr.c index 9ea4fd1..94372f2 100644 --- a/lustre/osd-zfs/osd_xattr.c +++ b/lustre/osd-zfs/osd_xattr.c @@ -435,7 +435,7 @@ int __osd_sa_xattr_set(const struct lu_env *env, struct osd_object *obj, return rc; } -static int +int __osd_xattr_set(const struct lu_env *env, struct osd_object *obj, const struct lu_buf *buf, const char *name, int fl, struct osd_thandle *oh) @@ -549,8 +549,8 @@ out: } int osd_xattr_set(const struct lu_env *env, struct dt_object *dt, - const struct lu_buf *buf, const char *name, int fl, - struct thandle *handle, struct lustre_capa *capa) + const struct lu_buf *buf, const char *name, int fl, + struct thandle *handle, struct lustre_capa *capa) { struct osd_object *obj = osd_dt_obj(dt); struct osd_thandle *oh; @@ -567,10 +567,7 @@ int osd_xattr_set(const struct lu_env *env, struct dt_object *dt, cfs_down(&obj->oo_guard); CDEBUG(D_INODE, "Setting xattr %s with size %d\n", name, (int)buf->lb_len); - rc = __osd_sa_xattr_set(env, obj, buf, name, fl, oh); - /* place xattr in dnode if SA is full */ - if (rc == -EFBIG) - rc = __osd_xattr_set(env, obj, buf, name, fl, oh); + rc = osd_xattr_set_internal(env, obj, buf, name, fl, oh, capa); cfs_up(&obj->oo_guard); RETURN(rc); diff --git a/lustre/ost/ost_handler.c b/lustre/ost/ost_handler.c index 37c0304..61e0ba3 100644 --- a/lustre/ost/ost_handler.c +++ b/lustre/ost/ost_handler.c @@ -555,9 +555,20 @@ static __u32 ost_checksum_bulk(struct ptlrpc_bulk_desc *desc, int opc, OBD_FAIL_CHECK(OBD_FAIL_OST_CHECKSUM_RECEIVE)) { int off = desc->bd_iov[i].kiov_offset & ~CFS_PAGE_MASK; int len = desc->bd_iov[i].kiov_len; + struct page *np = cfs_alloc_page(CFS_ALLOC_STD); char *ptr = kmap(desc->bd_iov[i].kiov_page) + off; - memcpy(ptr, "bad3", min(4, len)); - kunmap(desc->bd_iov[i].kiov_page); + + if (np) { + char *ptr2 = kmap(np) + off; + + memcpy(ptr2, ptr, len); + memcpy(ptr2, "bad3", min(4, len)); + kunmap(np); + cfs_page_unpin(desc->bd_iov[i].kiov_page); + desc->bd_iov[i].kiov_page = np; + } else { + CERROR("can't alloc page for corruption\n"); + } } cfs_crypto_hash_update_page(hdesc, desc->bd_iov[i].kiov_page, desc->bd_iov[i].kiov_offset & ~CFS_PAGE_MASK, @@ -569,11 +580,20 @@ static __u32 ost_checksum_bulk(struct ptlrpc_bulk_desc *desc, int opc, OBD_FAIL_CHECK(OBD_FAIL_OST_CHECKSUM_SEND)) { int off = desc->bd_iov[i].kiov_offset & ~CFS_PAGE_MASK; int len = desc->bd_iov[i].kiov_len; + struct page *np = cfs_alloc_page(CFS_ALLOC_STD); char *ptr = kmap(desc->bd_iov[i].kiov_page) + off; - memcpy(ptr, "bad4", min(4, len)); - kunmap(desc->bd_iov[i].kiov_page); - /* nobody should use corrupted page again */ - ClearPageUptodate(desc->bd_iov[i].kiov_page); + + if (np) { + char *ptr2 = kmap(np) + off; + + memcpy(ptr2, ptr, len); + memcpy(ptr2, "bad4", min(4, len)); + kunmap(np); + cfs_page_unpin(desc->bd_iov[i].kiov_page); + desc->bd_iov[i].kiov_page = np; + } else { + CERROR("can't alloc page for corruption\n"); + } } } diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index f1ec2de..a75f054 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -19,6 +19,10 @@ if [ "$FAILURE_MODE" = "HARD" ]; then ALWAYS_EXCEPT="$ALWAYS_EXCEPT $CONFIG_EXCEPTIONS" fi +# LU-2059 +ALWAYS_EXCEPT="$ALWAYS_EXCEPT 5d 19b 21b 27a" + + SRCDIR=`dirname $0` PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH @@ -558,6 +562,11 @@ is_blkdev () { # test_17() { + if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" + return + fi + setup check_mount || return 41 cleanup || return $? @@ -1191,6 +1200,11 @@ cleanup_32() { } test_32a() { + if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" + return + fi + client_only && skip "client only testing" && return 0 [ "$NETTYPE" = "tcp" ] || { skip "NETTYPE != tcp" && return 0; } [ -z "$TUNEFS" ] && skip_env "No tunefs" && return 0 @@ -1258,6 +1272,11 @@ test_32a() { run_test 32a "Upgrade from 1.8 (not live)" test_32b() { + if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" + return + fi + client_only && skip "client only testing" && return 0 [ "$NETTYPE" = "tcp" ] || { skip "NETTYPE != tcp" && return 0; } [ -z "$TUNEFS" ] && skip_env "No tunefs" && return @@ -1691,6 +1710,11 @@ test_37() { run_test 37 "verify set tunables works for symlink device" test_38() { # bug 14222 + if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" + return + fi + setup # like runtests COUNT=10 @@ -2441,6 +2465,11 @@ diff_files_xattrs() } test_52() { + if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" + return + fi + start_mds [ $? -eq 0 ] || { error "Unable to start MDS"; return 1; } start_ost @@ -2646,6 +2675,11 @@ test_53b() { run_test 53b "check MDT thread count params" test_54a() { + if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" + return + fi + do_rpc_nodes $(facet_host ost1) run_llverdev $(ostdevname 1) -p [ $? -eq 0 ] || error "llverdev failed!" reformat_and_config @@ -2653,6 +2687,11 @@ test_54a() { run_test 54a "test llverdev and partial verify of device" test_54b() { + if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" + return + fi + setup run_llverfs $MOUNT -p [ $? -eq 0 ] || error "llverfs failed!" @@ -2667,6 +2706,11 @@ lov_objid_size() } test_55() { + if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" + return + fi + local mdsdev=$(mdsdevname 1) local mdsvdev=$(mdsvdevname 1) @@ -2742,8 +2786,8 @@ count_osts() { } test_58() { # bug 22658 - if [ $(facet_fstype mds) == zfs ]; then - skip "Does not work with ZFS-based MDTs yet" + if [ $(facet_fstype mds) != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" return fi setup_noconfig @@ -2890,6 +2934,11 @@ test_61() { # LU-80 run_test 61 "large xattr" test_62() { + if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" + return + fi + # MRP-118 local mdsdev=$(mdsdevname 1) local ostdev=$(ostdevname 1) diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index c31107c..10b1621 100644 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -3985,6 +3985,11 @@ run_test 56w "check lfs_migrate -c stripe_count works" test_57a() { # note test will not do anything if MDS is not local + if [ "$(facet_type_fstype MDS)" != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" + return + fi + remote_mds_nodsh && skip "remote MDS with nodsh" && return local MNTDEV="osd*.*MDT*.mntdev" DEV=$(do_facet $SINGLEMDS lctl get_param -n $MNTDEV) @@ -4000,6 +4005,11 @@ test_57a() { run_test 57a "verify MDS filesystem created with large inodes ==" test_57b() { + if [ "$(facet_type_fstype MDS)" != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" + return + fi + remote_mds_nodsh && skip "remote MDS with nodsh" && return local dir=$DIR/d57b diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 9afb3ea..aa17bb8 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -455,6 +455,7 @@ load_modules_local() { grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; } [ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then + modprobe zfs load_module osd-zfs/osd_zfs fi load_module mgs/mgs -- 1.8.3.1