From febcbb91e256efd07e4f2bcd58c7ad1bcfb52e00 Mon Sep 17 00:00:00 2001 From: Andreas Dilger Date: Wed, 25 Jul 2018 19:47:51 -0600 Subject: [PATCH] LUDOC-249 migrate: Improve lfs_migrate documentation List the new method of max_create_count=0 for disabling the MDS to create new objects on OSTs during migration. This avoids issues when using "deactivate" where the OST objects were not being deleted as objects are being moved off of the OST. Separate the directory migration from the data migration more clearly. Signed-off-by: Andreas Dilger Change-Id: Ic8fa99271d8d543ec93ddac3917c8a7cad49973b Reviewed-on: https://review.whamcloud.com/32882 Tested-by: Jenkins Reviewed-by: Joseph Gmitter --- LustreMaintenance.xml | 31 ++++++---- Makefile | 4 +- ManagingFileSystemIO.xml | 150 ++++++++++++++++------------------------------- UserUtilities.xml | 24 +++++--- 4 files changed, 87 insertions(+), 122 deletions(-) diff --git a/LustreMaintenance.xml b/LustreMaintenance.xml index 12b8267..10b618c 100644 --- a/LustreMaintenance.xml +++ b/LustreMaintenance.xml @@ -420,7 +420,7 @@ oss# mount -t lustre /dev/sda /mnt/testfs/ost12 This redistributes file data over the entire set of OSTs. For example, to rebalance all files within the directory /mnt/lustre/dir, enter: - client# lfs_migrate /mnt/lustre/file + client# lfs_migrate /mnt/lustre/dir To migrate files within the /test file system on OST0004 that are larger than 4GB in size to other OSTs, enter: @@ -474,16 +474,16 @@ Removing and Restoring MDTs and OSTs
<indexterm><primary>maintenance</primary><secondary>removing an MDT</secondary></indexterm>Removing an MDT from the File System If the MDT is permanently inaccessible, - lfs rm_entry {directory} can be used to delete the - directory entry for the unavailable MDT. Using rmdir - would otherwise report an IO error due to the remote MDT being inactive. - Please note that if the MDT is available, standard - rm -r should be used to delete the remote directory. - After the remote directory has been removed, the administrator should - mark the MDT as permanently inactive with: -lctl conf_param {MDT name}.mdc.active=0 -A user can identify which MDT holds a remote sub-directory using -the lfs utility. For example: + lfs rm_entry {directory} can be used to delete the + directory entry for the unavailable MDT. Using rmdir + would otherwise report an IO error due to the remote MDT being inactive. + Please note that if the MDT is available, standard + rm -r should be used to delete the remote directory. + After the remote directory has been removed, the administrator should + mark the MDT as permanently inactive with: + lctl conf_param {MDT name}.mdc.active=0 + A user can identify which MDT holds a remote sub-directory using + the lfs utility. For example: client$ lfs getstripe --mdt-index /mnt/lustre/remote_dir1 1 client$ mkdir /mnt/lustre/local_dir0 @@ -492,8 +492,8 @@ client$ lfs getstripe --mdt-index /mnt/lustre/local_dir0 The lfs getstripe --mdt-index command returns the index of the MDT that is serving the given directory. -
-
+
+
<indexterm><primary>maintenance</primary></indexterm> <indexterm><primary>maintenance</primary><secondary>inactive MDTs</secondary></indexterm>Working with Inactive MDTs @@ -570,6 +570,11 @@ client$ lfs getstripe --mdt-index /mnt/lustre/local_dir0 files with objects on the deactivated OST, and copy them to other OSTs in the file system to: client# lfs find --ost ost_name /mount/point | lfs_migrate -y + Note that if multiple OSTs are being deactivated at one + time, the lfs find command can take multiple + --ost arguments, and will return files that + are located on any of the specified OSTs. + If the OST is no longer available, delete the files diff --git a/Makefile b/Makefile index 94fa6d1..4fee398 100644 --- a/Makefile +++ b/Makefile @@ -26,12 +26,14 @@ XSL_F16=/usr/share/sgml/docbook/xsl-ns-stylesheets XSL_SLE=/usr/share/xml/docbook/stylesheet/nwalsh5/current XSL_BREW=/usr/local/opt/docbook-xsl/docbook-xsl XSL_MAC=/opt/local/share/xsl/docbook-xsl +XSL_MAC2=/opt/local/share/xsl/docbook-xsl-nons XSL=$(or $(shell ls -d $(XSL_UBN) 2> /dev/null), \ $(shell ls -d $(XSL_REL) 2> /dev/null), \ $(shell ls -d $(XSL_F16) 2> /dev/null), \ $(shell ls -d $(XSL_SLE) 2> /dev/null), \ $(shell ls -d $(XSL_BREW) 2> /dev/null),\ - $(shell ls -d $(XSL_MAC) 2> /dev/null)) + $(shell ls -d $(XSL_MAC) 2> /dev/null),\ + $(shell ls -d $(XSL_MAC2) 2> /dev/null)) PRIMARYXSL=$(XSL)/$(subst $(TGT_BASE).,,$@)/docbook.xsl PRIMARYCHUNKXSL=$(XSL)/html/chunkfast.xsl diff --git a/ManagingFileSystemIO.xml b/ManagingFileSystemIO.xml index 51e2cb1..c00c63c 100644 --- a/ManagingFileSystemIO.xml +++ b/ManagingFileSystemIO.xml @@ -15,12 +15,13 @@ xml:id="managingfilesystemio"> Handling Full OSTs Sometimes a Lustre file system becomes unbalanced, often due to incorrectly-specified stripe settings, or when very large files are created - that are not striped over all of the OSTs. If an OST is full and an attempt - is made to write more information to the file system, an error occurs. The - procedures below describe how to handle a full OST. + that are not striped over all of the OSTs. Lustre will automatically avoid + allocating new files on OSTs that are full. If an OST is completely full and + more data is written to files already located on that OST, an error occurs. + The procedures below describe how to handle a full OST. The MDS will normally handle space balancing automatically at file - creation time, and this procedure is normally not needed, but may be - desirable in certain circumstances (e.g. when creating very large files + creation time, and this procedure is normally not needed, but manual data + migration may be desirable in some cases (e.g. creating very large files that would consume more than the total free space of the full OSTs).
@@ -67,72 +68,28 @@ dd: writing '/mnt/testfs/test_3': No space left on device <title> <indexterm> <primary>I/O</primary> - <secondary>taking OST offline</secondary> - </indexterm>Taking a Full OST Offline + disabling OST creates + Disabling creates on a Full OST To avoid running out of space in the file system, if the OST usage is imbalanced and one or more OSTs are close to being full while there - are others that have a lot of space, the full OSTs may optionally be - deactivated at the MDS to prevent the MDS from allocating new objects - there. + are others that have a lot of space, the MDS will typically avoid file + creation on the full OST(s) automatically. The full OSTs may optionally + be deactivated manually on the MDS to ensure the MDS will not allocate + new objects there. - Log into the MDS server: - -client# ssh root@192.168.0.10 -root@192.168.0.10's password: -Last login: Wed Nov 26 13:35:12 2008 from 192.168.0.6 - - - - Use the - lctl dl command to show the status of all file - system components: - -mds# lctl dl -0 UP mgs MGS MGS 9 -1 UP mgc MGC192.168.0.10@tcp e384bb0e-680b-ce25-7bc9-81655dd1e813 5 -2 UP mdt MDS MDS_uuid 3 -3 UP lov testfs-mdtlov testfs-mdtlov_UUID 4 -4 UP mds testfs-MDT0000 testfs-MDT0000_UUID 5 -5 UP osc testfs-OST0000-osc testfs-mdtlov_UUID 5 -6 UP osc testfs-OST0001-osc testfs-mdtlov_UUID 5 -7 UP osc testfs-OST0002-osc testfs-mdtlov_UUID 5 -8 UP osc testfs-OST0003-osc testfs-mdtlov_UUID 5 -9 UP osc testfs-OST0004-osc testfs-mdtlov_UUID 5 -10 UP osc testfs-OST0005-osc testfs-mdtlov_UUID 5 - - - - Use - lctl deactivate to take the full OST - offline: - -mds# lctl --device 7 deactivate - - - - Display the status of the file system components: + Log into the MDS server and use the lctl + command to stop new object creation on the full OST(s): + -mds# lctl dl -0 UP mgs MGS MGS 9 -1 UP mgc MGC192.168.0.10@tcp e384bb0e-680b-ce25-7bc9-81655dd1e813 5 -2 UP mdt MDS MDS_uuid 3 -3 UP lov testfs-mdtlov testfs-mdtlov_UUID 4 -4 UP mds testfs-MDT0000 testfs-MDT0000_UUID 5 -5 UP osc testfs-OST0000-osc testfs-mdtlov_UUID 5 -6 UP osc testfs-OST0001-osc testfs-mdtlov_UUID 5 -7 IN osc testfs-OST0002-osc testfs-mdtlov_UUID 5 -8 UP osc testfs-OST0003-osc testfs-mdtlov_UUID 5 -9 UP osc testfs-OST0004-osc testfs-mdtlov_UUID 5 -10 UP osc testfs-OST0005-osc testfs-mdtlov_UUID 5 +mds# lctl set_param osp.fsname-OSTnnnn*.max_create_count=0 - The device list shows that OST0002 is now inactive. When new files - are created in the file system, they will only use the remaining active - OSTs. Either manual space rebalancing can be done by migrating data to - other OSTs, as shown in the next section, or normal file deletion and - creation can be allowed to passively rebalance the space usage. + When new files are created in the file system, they will only use + the remaining OSTs. Either manual space rebalancing can be done by + migrating data to other OSTs, as shown in the next section, or normal + file deletion and creation can passively rebalance the space usage.
@@ -141,13 +98,38 @@ mds# lctl dl <secondary>migrating data</secondary> </indexterm> <indexterm> - <primary>migrating metadata</primary> - </indexterm> - <indexterm> <primary>maintenance</primary> <secondary>full OSTs</secondary> </indexterm>Migrating Data within a File System + If there is a need to move the file data from the current + OST(s) to new OST(s), the data must be migrated (copied) + to the new location. The simplest way to do this is to use the + lfs_migrate command, as described in + . +
+
+ + <indexterm> + <primary>I/O</primary> + <secondary>bringing OST online</secondary> + </indexterm> + <indexterm> + <primary>maintenance</primary> + <secondary>bringing OST online</secondary> + </indexterm>Returning an Inactive OST Back Online + Once the full OST(s) no longer are severely imbalanced, due + to either active or passive data redistribution, they should be + reactivated so they will again have new files allocated on them. + +[mds]# lctl set_param osp.testfs-OST0002.max_create_count=20000 + +
+
+ + <indexterm> + <primary>migrating metadata</primary> + </indexterm>Migrating Directories to a new MDT Lustre software version 2.8 includes a feature to migrate metadata (directories and inodes therein) between MDTs. This migration can only be performed on whole directories. For example, @@ -172,44 +154,10 @@ $ for i in $(seq 3); do lfs getstripe -M ./testremote/${i}.txt; done For more information, see man lfs-migrate Currently, only whole directories can be migrated between MDTs. During migration each file receives a new identifier - (FID). As a consequence, the file receives a new inode number. Some + (FID). As a consequence, the file will report a new inode number. Some system tools (for example, backup and archiving tools) may consider the migrated files to be new, even though the contents are unchanged. - If there is a need to migrate the file data - from the current OST(s) to new OST(s), the data must be migrated (copied) - to the new location. The simplest way to do this is to use the - lfs_migrate command, see - . -
-
- - <indexterm> - <primary>I/O</primary> - <secondary>bringing OST online</secondary> - </indexterm> - <indexterm> - <primary>maintenance</primary> - <secondary>bringing OST online</secondary> - </indexterm>Returning an Inactive OST Back Online - Once the deactivated OST(s) no longer are severely imbalanced, due - to either active or passive data redistribution, they should be - reactivated so they will again have new files allocated on them. - -[mds]# lctl --device 7 activate -[mds]# lctl dl - 0 UP mgs MGS MGS 9 - 1 UP mgc MGC192.168.0.10@tcp e384bb0e-680b-ce25-7bc9-816dd1e813 5 - 2 UP mdt MDS MDS_uuid 3 - 3 UP lov testfs-mdtlov testfs-mdtlov_UUID 4 - 4 UP mds testfs-MDT0000 testfs-MDT0000_UUID 5 - 5 UP osc testfs-OST0000-osc testfs-mdtlov_UUID 5 - 6 UP osc testfs-OST0001-osc testfs-mdtlov_UUID 5 - 7 UP osc testfs-OST0002-osc testfs-mdtlov_UUID 5 - 8 UP osc testfs-OST0003-osc testfs-mdtlov_UUID 5 - 9 UP osc testfs-OST0004-osc testfs-mdtlov_UUID 5 - 10 UP osc testfs-OST0005-osc testfs-mdtlov_UUID -
diff --git a/UserUtilities.xml b/UserUtilities.xml index d1f5c89..b60ba9e 100644 --- a/UserUtilities.xml +++ b/UserUtilities.xml @@ -1085,7 +1085,7 @@ $ lfs setstripe --pool my_pool /mnt/lustre/dir Synopsis lfs_migrate [lfs_setstripe_options] - [-h] [-n] [-q] [-R] [-s] [-y] [file|directory ...] + [-h] [-n] [-q] [-R] [-s] [-y] [-0] [file|directory ...]
@@ -1258,21 +1258,31 @@ lfs_migrate [lfs_setstripe_options] scripts, use with caution). + + + -0 + + Expect NUL-terminated filenames on standard input, as + generated by lfs find -print0 or + find -print0. This allows filenames with + embedded newlines to be handled correctly. + +
Examples - Rebalances all files in - /mnt/lustre/dir. + Rebalance all files in + /mnt/lustre/dir: -$ lfs_migrate /mnt/lustre/file +$ lfs_migrate /mnt/lustre/dir - Migrates files in /test filesystem on OST004 larger than 4 GB in - size. + Migrate files in /test filesystem on OST0004 larger than 4 GB in + size and older than a day old: -$ lfs find /test -obd test-OST004 -size +4G | lfs_migrate -y +$ lfs find /test -obd test-OST0004 -size +4G -mtime +1 | lfs_migrate -y
-- 1.8.3.1