-FIGURES = figures/ost-setattr-generic.png \
- figures/ost-punch-generic.png \
- figures/mds-reint-setattr-generic.png \
- figures/mds-reint-setxattr-generic.png \
- figures/mds-getxattr-generic.png \
- figures/ldlm-enqueue-generic.png \
- figures/ldlm-enqueue-intent-layout-generic.png \
- figures/ldlm-bl-callback-generic.png \
- figures/ldlm-cp-callback-generic.png \
- figures/ldlm-cancel-generic.png \
- figures/chmod_rpcs.png \
+FIGURES = figures/chmod_rpcs.png \
figures/mds-reint-setattr-request.png \
figures/mds-reint-setattr-reply.png \
figures/mds-reint-setxattr-request.png \
figures/ldlm-enqueue-reply.png \
figures/ldlm-gl-callback-request.png \
figures/ldlm-gl-callback-reply.png \
- figures/ldlm-enqueue-intent-getattr-generic.png \
- figures/ldlm-gl-callback-generic.png \
figures/statfs_rpcs.png \
figures/mds-statfs-request.png \
figures/mds-statfs-reply.png \
figures/ost-statfs-request.png \
figures/ost-statfs-reply.png \
- figures/mds-statfs-generic.png \
- figures/ost-statfs-generic.png \
figures/getxattr_rpcs.png \
figures/ldlm-enqueue-intent-getxattr-request.png \
figures/ldlm-enqueue-intent-getxattr-reply.png \
- figures/ldlm-enqueue-intent-getxattr-generic.png \
figures/setxattr_rpcs.png \
figures/client_mgs_connect_rpcs.png \
figures/client_mdt_connect_rpcs.png \
figures/client_ost_connect_rpcs.png \
figures/umount_rpcs.png \
- figures/ost-connect-generic.png \
figures/ost-connect-request.png \
figures/ost-connect-reply.png \
- figures/mds-connect-generic.png \
figures/mds-connect-request.png \
figures/mds-connect-reply.png \
- figures/mgs-connect-generic.png \
figures/mgs-connect-request.png \
figures/mgs-connect-reply.png \
- figures/ost-disconnect-generic.png \
- figures/mds-disconnect-generic.png \
- figures/mgs-disconnect-generic.png \
- figures/mds-getattr-generic.png \
- figures/mds-getstatus-generic.png \
- figures/mgs-config-read-generic.png \
- figures/llog-origin-handle-create-generic.png \
+ figures/ost-disconnect-request.png \
+ figures/ost-disconnect-reply.png \
+ figures/mds-disconnect-request.png \
+ figures/mds-disconnect-reply.png \
+ figures/mgs-disconnect-request.png \
+ figures/mgs-disconnect-reply.png \
+ figures/mds-getattr-request.png \
+ figures/mds-getattr-reply.png \
+ figures/mds-getstatus-request.png \
+ figures/mds-getstatus-reply.png \
+ figures/mgs-config-read-reply.png \
+ figures/mgs-config-read-request.png \
figures/llog-origin-handle-create-reply.png \
figures/llog-origin-handle-create-request.png \
- figures/llog-origin-handle-next-block-generic.png \
figures/llog-origin-handle-next-block-request.png \
figures/llog-origin-handle-next-block-reply.png \
- figures/llog-origin-handle-read-header-generic.png \
figures/llog-origin-handle-read-header-request.png \
figures/llog-origin-handle-read-header-reply.png
TEXT = protocol.txt \
introduction.txt \
- transno.txt \
+ client.txt \
+ target.txt \
+ rpc.txt \
connection.txt \
struct_obd_connect_data.txt \
- import.txt \
- export.txt \
+ struct_obd_import.txt \
+ struct_obd_export.txt \
struct_obd_uuid.txt \
timeouts.txt \
eviction.txt \
recovery.txt \
+ transno.txt \
path_lookup.txt \
lov_index.txt \
grant.txt \
ost_setattr.txt \
struct_ptlrpc_body.txt \
struct_lustre_handle.txt \
+ struct_ost_body.txt \
ost_connect.txt \
ost_disconnect.txt \
ost_punch.txt \
3015 1875 3015 1950 3015 2025 3015 2100 3015 2175 3015 2250
3015 2325 3015 2400 3015 2475
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 4350 825 4350 900 4350 975 4350 1050 4350 1125 4350 1200
- 4350 1275 4350 1350 4350 1425
+ 4575 825 4575 900 4575 975 4575 1050 4575 1125 4575 1200
+ 4575 1275 4575 1350 4575 1425
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 5775 825 5775 900 5775 975 5775 1050 5775 1125 5775 1200
- 5775 1275 5775 1350 5775 1425
+ 6075 825 6075 900 6075 975 6075 1050 6075 1125 6075 1200
+ 6075 1275 6075 1350 6075 1425
2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
- 2025 835 10350 840 10335 1425 1200 1435 1200 985
+ 2025 822 10350 827 10335 1412 1200 1422 1200 972
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 7650 825 7650 900 7650 975 7650 1050 7650 1125 7650 1200
- 7650 1275 7650 1350 7650 1425
+ 7875 825 7875 900 7875 975 7875 1050 7875 1125 7875 1200
+ 7875 1275 7875 1350 7875 1425
4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
-4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
4 0 0 50 -1 16 18 0.0000 4 270 615 1140 1950 reply\001
4 0 0 50 -1 16 18 0.0000 4 270 1530 1365 2250 ptlrpc_body\001
-4 0 0 50 -1 16 18 0.0000 4 270 1185 3105 1320 obd_uuid\001
-4 0 0 50 -1 16 18 0.0000 4 270 1185 4500 1275 obd_uuid\001
-4 0 0 50 -1 16 18 0.0000 4 270 2415 7770 1275 obd_connect_data\001
-4 0 0 50 -1 16 18 0.0000 4 270 1680 5895 1275 lustre_handle\001
4 0 0 50 -1 16 18 0.0000 4 270 2415 3150 2250 obd_connect_data\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1275 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1440 3075 1275 target_uuid\001
+4 0 0 50 -1 16 18 0.0000 4 270 1350 4650 1275 client_uuid\001
+4 0 0 50 -1 16 18 0.0000 4 270 1680 6150 1275 lustre_handle\001
+4 0 0 50 -1 16 18 0.0000 4 270 2415 7950 1275 obd_connect_data\001
4 0 0 50 -1 16 18 0.0000 4 270 2250 1050 675 MDS_CONNECT\001
4 0 0 50 -1 16 18 0.0000 4 270 1530 1365 2250 ptlrpc_body\001
4 0 0 50 -1 16 18 0.0000 4 270 2940 1050 675 MGS_CONFIG_READ\001
4 0 0 50 -1 16 18 0.0000 4 270 2295 3105 1320 mgs_config_body\001
-4 0 0 50 -1 16 18 0.0000 4 270 2295 3075 2250 mgs_config_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 2025 3075 2250 mgs_config_res\001
3015 1875 3015 1950 3015 2025 3015 2100 3015 2175 3015 2250
3015 2325 3015 2400 3015 2475
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 4350 825 4350 900 4350 975 4350 1050 4350 1125 4350 1200
- 4350 1275 4350 1350 4350 1425
+ 4575 825 4575 900 4575 975 4575 1050 4575 1125 4575 1200
+ 4575 1275 4575 1350 4575 1425
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 5775 825 5775 900 5775 975 5775 1050 5775 1125 5775 1200
- 5775 1275 5775 1350 5775 1425
+ 6075 825 6075 900 6075 975 6075 1050 6075 1125 6075 1200
+ 6075 1275 6075 1350 6075 1425
2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
- 2025 835 10350 840 10335 1425 1200 1435 1200 985
+ 2025 822 10350 827 10335 1412 1200 1422 1200 972
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 7650 825 7650 900 7650 975 7650 1050 7650 1125 7650 1200
- 7650 1275 7650 1350 7650 1425
+ 7875 825 7875 900 7875 975 7875 1050 7875 1125 7875 1200
+ 7875 1275 7875 1350 7875 1425
4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
-4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
4 0 0 50 -1 16 18 0.0000 4 270 615 1140 1950 reply\001
4 0 0 50 -1 16 18 0.0000 4 270 1530 1365 2250 ptlrpc_body\001
-4 0 0 50 -1 16 18 0.0000 4 270 1185 3105 1320 obd_uuid\001
-4 0 0 50 -1 16 18 0.0000 4 270 1185 4500 1275 obd_uuid\001
-4 0 0 50 -1 16 18 0.0000 4 270 2415 7770 1275 obd_connect_data\001
-4 0 0 50 -1 16 18 0.0000 4 270 1680 5895 1275 lustre_handle\001
4 0 0 50 -1 16 18 0.0000 4 270 2415 3150 2250 obd_connect_data\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1275 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1440 3075 1275 target_uuid\001
+4 0 0 50 -1 16 18 0.0000 4 270 1350 4650 1275 client_uuid\001
+4 0 0 50 -1 16 18 0.0000 4 270 1680 6150 1275 lustre_handle\001
+4 0 0 50 -1 16 18 0.0000 4 270 2415 7950 1275 obd_connect_data\001
4 0 0 50 -1 16 18 0.0000 4 270 2265 1050 675 MGS_CONNECT\001
3015 1875 3015 1950 3015 2025 3015 2100 3015 2175 3015 2250
3015 2325 3015 2400 3015 2475
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 4350 825 4350 900 4350 975 4350 1050 4350 1125 4350 1200
- 4350 1275 4350 1350 4350 1425
+ 4575 825 4575 900 4575 975 4575 1050 4575 1125 4575 1200
+ 4575 1275 4575 1350 4575 1425
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 5775 825 5775 900 5775 975 5775 1050 5775 1125 5775 1200
- 5775 1275 5775 1350 5775 1425
+ 6075 825 6075 900 6075 975 6075 1050 6075 1125 6075 1200
+ 6075 1275 6075 1350 6075 1425
2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
- 2025 835 10350 840 10335 1425 1200 1435 1200 985
+ 2025 822 10350 827 10335 1412 1200 1422 1200 972
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 7650 825 7650 900 7650 975 7650 1050 7650 1125 7650 1200
- 7650 1275 7650 1350 7650 1425
+ 7875 825 7875 900 7875 975 7875 1050 7875 1125 7875 1200
+ 7875 1275 7875 1350 7875 1425
4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
-4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
4 0 0 50 -1 16 18 0.0000 4 270 615 1140 1950 reply\001
4 0 0 50 -1 16 18 0.0000 4 270 1530 1365 2250 ptlrpc_body\001
4 0 0 50 -1 16 18 0.0000 4 270 2205 1050 675 OST_CONNECT\001
-4 0 0 50 -1 16 18 0.0000 4 270 1185 3105 1320 obd_uuid\001
-4 0 0 50 -1 16 18 0.0000 4 270 1185 4500 1275 obd_uuid\001
-4 0 0 50 -1 16 18 0.0000 4 270 2415 7770 1275 obd_connect_data\001
-4 0 0 50 -1 16 18 0.0000 4 270 1680 5895 1275 lustre_handle\001
4 0 0 50 -1 16 18 0.0000 4 270 2415 3150 2250 obd_connect_data\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1275 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1440 3075 1275 target_uuid\001
+4 0 0 50 -1 16 18 0.0000 4 270 1350 4650 1275 client_uuid\001
+4 0 0 50 -1 16 18 0.0000 4 270 1680 6150 1275 lustre_handle\001
+4 0 0 50 -1 16 18 0.0000 4 270 2415 7950 1275 obd_connect_data\001
--- /dev/null
+Client
+~~~~~~
+
+Host systems that mount the Lustre file system and are generally
+refered to as "clients" of the file system services. Most, but not
+all, Lustre protocol actions (remote procedure calls, or RPCs) are
+initiated by clients.
The Lustre protocol is connection-based in that each two entities
maintain shared, coordinated state information. The most common
-example of two such entities are a client and a target on some
+example is the connection between a client and the target on some
server. The target is identified by name to the client through an
interaction with the management server. The client then 'connects' to
the given target on the indicated server by sending the appropriate
-version of the various *_CONNECT message (MGS_CONNECT, MDS_CONNECT, or
-OST_CONNECT) and receiving back the
-corresponding *_CONNECT reply. The server creates an 'export' for the
-connection between the target and the client, and the export holds the
-server state information for that connection. When the client gets the
-reply it creates an 'import', and the import holds the client state
-information for that connection. Note that if a server has N targets
-and M clients have connected to them, the server will have N x M
-exports and each client will have N imports.
+version of the various *_CONNECT messages (MGS_CONNECT, MDS_CONNECT,
+or OST_CONNECT) and receiving back the corresponding *_CONNECT
+reply. The server creates an 'export' (See <<struct-obd-export>>) for
+the connection between the target and the client, and the export holds
+the server state information for that connection. When the client gets
+the reply it creates an 'import' (See <<struct-obd-import>>), and the
+import holds the client state information for that connection. Note
+that if a server has N targets and M clients have connected to them,
+the server will have N x M exports and each client will have N
+imports.
There are also connections between the servers: Each MDS and OSS has a
connection to the MGS, where the MDS (respectively the OSS) plays the
role of the client in the above discussion. That is, the MDS initiates
the connection and has an import for the MGS, while the MGS has an
export for each MDS. Each MDS creates an 'object storage proxy' (OSP)
-connection to each OST, with an import on
-the MDS and an export on the OSS. This connection supports requests
-from the MDS to the OST to create and destroy data objects, to set
-attributes (such as permission bits), and for 'statfs' information for
-precreation needs. Each OSS also connects to the first MDS to get
-access to auxiliary services, with an import on the OSS and an export
-on the first MDS. The auxiliary services are: the File ID Location
-Database (FLDB), the quota master service, and the sequence
-controller. This connection for auxiliary services is a 'light weight
-proxy' (LWP) connection in that it has no replay functionality and
-consumes no space on the MDS for client data. Each MDS connects also
-to all other MDSs for DNE needs.
+connection to each OST, with an import on the MDS and an export on the
+OSS. This connection supports requests from the MDS to the OST to
+create and destroy data objects, to set attributes (such as permission
+bits), and for 'statfs' information for precreation needs. Each OSS
+also connects to the first MDS to get access to auxiliary services,
+with an import on the OSS and an export on the first MDS. The
+auxiliary services are: the File ID Location Database (FLDB), the
+quota master service, and the sequence controller. This connection for
+auxiliary services is a 'light weight proxy' (LWP) connection in that
+it has no replay functionality (See <<recovery>>) and consumes no
+space on the MDS for client data. Each MDS also connects to all other
+MDSs for DNE needs.
Finally, for some communications the roles of message initiation and
message reply are reversed. This is the case, for instance, with
include::struct_obd_connect_data.txt[]
-include::import.txt[]
+include::struct_obd_import.txt[]
-include::export.txt[]
+include::struct_obd_export.txt[]
include::timeouts.txt[]
+++ /dev/null
-Lustre Protocol Documentation - setattr
-=======================================
-Andrew Uselton <andrew.c.uselton@intel.com>
-v0.0, May 2015
-:author: Andrew Uselton
-:doctype: article
-:Author Initials: ACU
-:toc:
-:icons:
-:numbered:
-:imagesdir: ./figures
-:website: http://lustre.org/
-:keywords: PtlRPC, Lustre, Protocol
-
-
-include::setattr.txt[]
-
-:numbered!:
-
-[appendix]
-License
--------
-
-Copyright (C) Intel 2015
-
-This work is licensed under a Creative Commons Attribution-ShareAlike
-4.0 International License (CC BY-SA 4.0). See
-<https://creativecommons.org/licenses/by-sa/4.0/> for more detail.
-
+++ /dev/null
-Lustre Protocol Documentation - getattr
-=======================================
-Andrew Uselton <andrew.c.uselton@intel.com>
-v0.0, May 2015
-:author: Andrew Uselton
-:doctype: article
-:Author Initials: ACU
-:toc:
-:icons:
-:numbered:
-:imagesdir: ./figures
-:website: http://lustre.org/
-:keywords: PtlRPC, Lustre, Protocol
-
-File System Operations
-----------------------
-
-include::getattr.txt[]
-
-RPCs
-----
-[[rpcs]]
-
-include::ldlm_enqueue.txt[]
-
-include::ldlm_gl_callback.txt[]
-
-:numbered!:
-
-[appendix]
-License
--------
-
-Copyright (C) Intel 2015
-
-This work is licensed under a Creative Commons Attribution-ShareAlike
-4.0 International License (CC BY-SA 4.0). See
-<https://creativecommons.org/licenses/by-sa/4.0/> for more detail.
-
+++ /dev/null
-Lustre Protocol Documentation - setattr
-=======================================
-Andrew Uselton <andrew.c.uselton@intel.com>
-v0.0, May 2015
-:author: Andrew Uselton
-:doctype: article
-:Author Initials: ACU
-:toc:
-:icons:
-:numbered:
-:imagesdir: ./figures
-:website: http://lustre.org/
-:keywords: PtlRPC, Lustre, Protocol
-
-include::setattr_terms.txt[]
-
-File System Operations
-----------------------
-
-include::setattr.txt[]
-
-//////////////////////////////////////////////////////////////////////
-This is back up one level (starting with ----):
-//////////////////////////////////////////////////////////////////////
-
-include::early_lock_cancellation.txt[]
-
-RPCs
-----
-[[rpcs]]
-
-include::ost_setattr.txt[]
-
-include::ost_punch.txt[]
-
-include::mds_reint.txt[]
-
-include::mds_getxattr.txt[]
-
-include::ldlm_enqueue.txt[]
-
-include::ldlm_bl_callback.txt[]
-
-include::ldlm_cp_callback.txt[]
-
-Data Structures and Defines
----------------------------
-[[data-structs]]
-
-Note that 'lustre_msg' and 'ptlrpc_body' are described in the main
-protocol document. Those details are not repeated in this brief
-extract. Only those structures that are particular to the 'setattr'
-activity are included here. This extract will be incorporated back in
-the protocol document.
-
-Miscellaneous Structured Data Types
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-<<struct-lu-fid>>
-
-include::lustre_handle.txt[]
-
-LDLM Structures
-~~~~~~~~~~~~~~~
-
-include::layout_intent.txt[]
-
-include::ldlm_resource_id.txt[]
-
-include::ldlm_intent.txt[]
-
-include::ldlm_resource_desc.txt[]
-
-include::ldlm_lock_desc.txt[]
-
-include::ldlm_request.txt[]
-
-include::ldlm_reply.txt[]
-
-include::ost_lvb.txt[]
-
-//////////////////////////////////////////////////////////////////////
-These are back up one level (starting with ~~~~):
-//////////////////////////////////////////////////////////////////////
-
-include::mds_reint_structs.txt[]
-
-include::ost_setattr_structs.txt[]
-
-:numbered!:
-
-[appendix]
-License
--------
-
-Copyright (C) Intel 2015
-
-This work is licensed under a Creative Commons Attribution-ShareAlike
-4.0 International License (CC BY-SA 4.0). See
-<https://creativecommons.org/licenses/by-sa/4.0/> for more detail.
-
-2
1200 2
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 3000 900 3000 975 3000 1050 3000 1125 3000 1200 3000 1275
- 3000 1350 3000 1425 3000 1500
+ 4575 825 4575 900 4575 975 4575 1050 4575 1125 4575 1200
+ 4575 1275 4575 1350 4575 1425
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 4350 825 4350 900 4350 975 4350 1050 4350 1125 4350 1200
- 4350 1275 4350 1350 4350 1425
-2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 5775 825 5775 900 5775 975 5775 1050 5775 1125 5775 1200
- 5775 1275 5775 1350 5775 1425
+ 6000 825 6000 900 6000 975 6000 1050 6000 1125 6000 1200
+ 6000 1275 6000 1350 6000 1425
2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
- 2025 835 10350 840 10335 1425 1200 1435 1200 985
+ 2100 822 10425 827 10410 1412 1275 1422 1275 972
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 3000 825 3000 900 3000 975 3000 1050 3000 1125 3000 1200
+ 3000 1275 3000 1350 3000 1425
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 7650 825 7650 900 7650 975 7650 1050 7650 1125 7650 1200
- 7650 1275 7650 1350 7650 1425
+ 7800 825 7800 900 7800 975 7800 1050 7800 1125 7800 1200
+ 7800 1275 7800 1350 7800 1425
4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
-4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
-4 0 0 50 -1 16 18 0.0000 4 270 1185 3105 1320 obd_uuid\001
-4 0 0 50 -1 16 18 0.0000 4 270 1185 4500 1275 obd_uuid\001
-4 0 0 50 -1 16 18 0.0000 4 270 2415 7770 1275 obd_connect_data\001
-4 0 0 50 -1 16 18 0.0000 4 270 1680 5895 1275 lustre_handle\001
+4 0 0 50 -1 16 18 0.0000 4 270 1350 4575 1275 client_uuid\001
+4 0 0 50 -1 16 18 0.0000 4 270 1440 3075 1275 target_uuid\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1275 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1680 6075 1275 lustre_handle\001
+4 0 0 50 -1 16 18 0.0000 4 270 2415 7875 1275 obd_connect_data\001
4 0 0 50 -1 16 18 0.0000 4 270 2250 1050 675 MDS_CONNECT\001
--- /dev/null
+#FIG 3.2 Produced by xfig version 3.2.5b
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
+ 1890 903 3195 888 3195 1503 1290 1503 1290 1053
+4 0 0 50 -1 16 18 0.0000 4 270 2745 1050 675 MDS_DISCONNECT\001
+4 0 0 50 -1 16 18 0.0000 4 270 615 1200 975 reply\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1425 1275 ptlrpc_body\001
--- /dev/null
+#FIG 3.2 Produced by xfig version 3.2.5b
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
+ 2025 975 3120 960 3120 1575 1215 1575 1215 1125
+4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 2745 1050 675 MDS_DISCONNECT\001
--- /dev/null
+#FIG 3.2 Produced by xfig version 3.2.5b
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
+ 1740 900 9975 900 9975 1500 1140 1500 1140 1050
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 2775 900 2775 975 2775 1050 2775 1125 2775 1200 2775 1275
+ 2775 1350 2775 1425 2775 1500
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 4200 900 4200 975 4200 1050 4200 1125 4200 1200 4200 1275
+ 4200 1350 4200 1425 4200 1500
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 5550 900 5550 975 5550 1050 5550 1125 5550 1200 5550 1275
+ 5550 1350 5550 1425 5550 1500
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 6225 900 6225 975 6225 1050 6225 1125 6225 1200 6225 1275
+ 6225 1350 6225 1425 6225 1500
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 7875 900 7875 975 7875 1050 7875 1125 7875 1200 7875 1275
+ 7875 1350 7875 1425 7875 1500
+4 0 0 50 -1 16 18 0.0000 4 270 2175 1050 675 MDS_GETATTR\001
+4 0 0 50 -1 16 18 0.0000 4 270 615 1125 975 reply\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1200 1275 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1305 2850 1275 mdt_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1245 4275 1275 MDT_MD\001
+4 0 0 50 -1 16 18 0.0000 4 210 570 5625 1275 ACL\001
+4 0 0 50 -1 16 18 0.0000 4 270 1275 6300 1275 fid1_capa\001
+4 0 0 50 -1 16 18 0.0000 4 270 1275 7950 1275 fid2_capa\001
--- /dev/null
+#FIG 3.2 Produced by xfig version 3.2.5b
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 3000 900 3000 975 3000 1050 3000 1125 3000 1200 3000 1275
+ 3000 1350 3000 1425 3000 1500
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 4545 915 4545 990 4545 1065 4545 1140 4545 1215 4545 1290
+ 4545 1365 4545 1440 4545 1515
+2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
+ 2025 900 6300 900 6300 1500 1200 1500 1200 1050
+4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1305 3105 1320 mdt_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1485 4680 1305 lustre_capa\001
+4 0 0 50 -1 16 18 0.0000 4 270 2175 1050 675 MDS_GETATTR\001
--- /dev/null
+#FIG 3.2 Produced by xfig version 3.2.5b
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
+ 1815 900 4575 900 4575 1500 1215 1500 1215 1050
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 2925 900 2925 975 2925 1050 2925 1125 2925 1200 2925 1275
+ 2925 1350 2925 1425 2925 1500
+4 0 0 50 -1 16 18 0.0000 4 270 2565 1050 675 MDS_GETSTATUS\001
+4 0 0 50 -1 16 18 0.0000 4 270 615 1200 975 reply\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1275 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1305 3000 1275 mdt_body\001
--- /dev/null
+#FIG 3.2 Produced by xfig version 3.2.5b
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 3000 900 3000 975 3000 1050 3000 1125 3000 1200 3000 1275
+ 3000 1350 3000 1425 3000 1500
+2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
+ 2025 900 4575 900 4575 1500 1200 1500 1200 1050
+4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1305 3105 1320 mdt_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 2565 1050 675 MDS_GETSTATUS\001
4545 915 4545 990 4545 1065 4545 1140 4545 1215 4545 1290
4545 1365 4545 1440 4545 1515
2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
- 2025 900 5505 900 5505 1485 1200 1500 1200 1050
+ 2025 900 8385 885 8385 1485 1200 1500 1200 1050
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 6225 930 6225 1005 6225 1080 6225 1155 6225 1230 6225 1305
+ 6225 1380 6225 1455 6225 1530
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 7155 915 7155 990 7155 1065 7155 1140 7155 1215 7155 1290
+ 7155 1365 7155 1440 7155 1515
4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
4 0 0 50 -1 16 18 0.0000 4 270 2370 1050 675 MDS_GETXATTR\001
4 0 0 50 -1 16 18 0.0000 4 270 1305 3105 1320 mdt_body\001
-4 0 0 50 -1 16 18 0.0000 4 150 705 4725 1320 name\001
+4 0 0 50 -1 16 18 0.0000 4 270 1485 4680 1305 lustre_capa\001
+4 0 0 50 -1 16 18 0.0000 4 150 705 6345 1320 name\001
+4 0 0 50 -1 16 18 0.0000 4 210 900 7275 1320 eadata\001
--- /dev/null
+#FIG 3.2 Produced by xfig version 3.2.5b
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
+ 1800 825 5460 825 5460 1425 1200 1425 1200 975
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 2850 825 2850 900 2850 975 2850 1050 2850 1125 2850 1200
+ 2850 1275 2850 1350 2850 1425
+4 0 0 50 -1 16 18 0.0000 4 270 2940 1050 675 MGS_CONFIG_READ\001
+4 0 0 50 -1 16 18 0.0000 4 270 615 1125 900 reply\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1275 1200 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 2025 2925 1200 mgs_config_res\001
--- /dev/null
+#FIG 3.2 Produced by xfig version 3.2.5b
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 3000 900 3000 975 3000 1050 3000 1125 3000 1200 3000 1275
+ 3000 1350 3000 1425 3000 1500
+2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
+ 2025 835 5475 825 5475 1425 1200 1435 1200 985
+4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 2940 1050 675 MGS_CONFIG_READ\001
+4 0 0 50 -1 16 18 0.0000 4 270 2295 3105 1320 mgs_config_body\001
-2
1200 2
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 3000 900 3000 975 3000 1050 3000 1125 3000 1200 3000 1275
- 3000 1350 3000 1425 3000 1500
+ 4575 825 4575 900 4575 975 4575 1050 4575 1125 4575 1200
+ 4575 1275 4575 1350 4575 1425
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 4350 825 4350 900 4350 975 4350 1050 4350 1125 4350 1200
- 4350 1275 4350 1350 4350 1425
-2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 5775 825 5775 900 5775 975 5775 1050 5775 1125 5775 1200
- 5775 1275 5775 1350 5775 1425
+ 6000 825 6000 900 6000 975 6000 1050 6000 1125 6000 1200
+ 6000 1275 6000 1350 6000 1425
2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
- 2025 835 10350 840 10335 1425 1200 1435 1200 985
+ 2100 822 10425 827 10410 1412 1275 1422 1275 972
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 3000 825 3000 900 3000 975 3000 1050 3000 1125 3000 1200
+ 3000 1275 3000 1350 3000 1425
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 7650 825 7650 900 7650 975 7650 1050 7650 1125 7650 1200
- 7650 1275 7650 1350 7650 1425
+ 7800 825 7800 900 7800 975 7800 1050 7800 1125 7800 1200
+ 7800 1275 7800 1350 7800 1425
4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
-4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
-4 0 0 50 -1 16 18 0.0000 4 270 1185 3105 1320 obd_uuid\001
-4 0 0 50 -1 16 18 0.0000 4 270 1185 4500 1275 obd_uuid\001
-4 0 0 50 -1 16 18 0.0000 4 270 2415 7770 1275 obd_connect_data\001
-4 0 0 50 -1 16 18 0.0000 4 270 1680 5895 1275 lustre_handle\001
+4 0 0 50 -1 16 18 0.0000 4 270 1350 4575 1275 client_uuid\001
+4 0 0 50 -1 16 18 0.0000 4 270 1440 3075 1275 target_uuid\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1275 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1680 6075 1275 lustre_handle\001
+4 0 0 50 -1 16 18 0.0000 4 270 2415 7875 1275 obd_connect_data\001
4 0 0 50 -1 16 18 0.0000 4 270 2265 1050 675 MGS_CONNECT\001
--- /dev/null
+#FIG 3.2 Produced by xfig version 3.2.5b
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
+ 1800 900 3105 885 3105 1500 1200 1500 1200 1050
+4 0 0 50 -1 16 18 0.0000 4 270 2760 1050 675 MGS_DISCONNECT\001
+4 0 0 50 -1 16 18 0.0000 4 270 615 1125 975 reply\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1275 ptlrpc_body\001
--- /dev/null
+#FIG 3.2 Produced by xfig version 3.2.5b
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
+ 2025 975 3120 960 3120 1575 1215 1575 1215 1125
+4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 2760 1050 675 MGS_DISCONNECT\001
-2
1200 2
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 3000 900 3000 975 3000 1050 3000 1125 3000 1200 3000 1275
- 3000 1350 3000 1425 3000 1500
+ 4575 825 4575 900 4575 975 4575 1050 4575 1125 4575 1200
+ 4575 1275 4575 1350 4575 1425
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 4350 825 4350 900 4350 975 4350 1050 4350 1125 4350 1200
- 4350 1275 4350 1350 4350 1425
-2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 5775 825 5775 900 5775 975 5775 1050 5775 1125 5775 1200
- 5775 1275 5775 1350 5775 1425
+ 6000 825 6000 900 6000 975 6000 1050 6000 1125 6000 1200
+ 6000 1275 6000 1350 6000 1425
2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
- 2025 835 10350 840 10335 1425 1200 1435 1200 985
+ 2100 822 10425 827 10410 1412 1275 1422 1275 972
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 3000 825 3000 900 3000 975 3000 1050 3000 1125 3000 1200
+ 3000 1275 3000 1350 3000 1425
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
- 7650 825 7650 900 7650 975 7650 1050 7650 1125 7650 1200
- 7650 1275 7650 1350 7650 1425
+ 7800 825 7800 900 7800 975 7800 1050 7800 1125 7800 1200
+ 7800 1275 7800 1350 7800 1425
4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
-4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
-4 0 0 50 -1 16 18 0.0000 4 270 1185 3105 1320 obd_uuid\001
-4 0 0 50 -1 16 18 0.0000 4 270 1185 4500 1275 obd_uuid\001
-4 0 0 50 -1 16 18 0.0000 4 270 2415 7770 1275 obd_connect_data\001
-4 0 0 50 -1 16 18 0.0000 4 270 1680 5895 1275 lustre_handle\001
4 0 0 50 -1 16 18 0.0000 4 270 2205 1050 675 OST_CONNECT\001
+4 0 0 50 -1 16 18 0.0000 4 270 1350 4575 1275 client_uuid\001
+4 0 0 50 -1 16 18 0.0000 4 270 1440 3075 1275 target_uuid\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1275 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1680 6075 1275 lustre_handle\001
+4 0 0 50 -1 16 18 0.0000 4 270 2415 7875 1275 obd_connect_data\001
--- /dev/null
+#FIG 3.2 Produced by xfig version 3.2.5b
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
+ 1800 900 3105 885 3105 1500 1200 1500 1200 1050
+4 0 0 50 -1 16 18 0.0000 4 270 2700 1050 675 OST_DISCONNECT\001
+4 0 0 50 -1 16 18 0.0000 4 270 615 1125 975 reply\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1275 ptlrpc_body\001
--- /dev/null
+#FIG 3.2 Produced by xfig version 3.2.5b
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
+ 2025 975 3120 960 3120 1575 1215 1575 1215 1125
+4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
+4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 2700 1050 675 OST_DISCONNECT\001
2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
3000 900 3000 975 3000 1050 3000 1125 3000 1200 3000 1275
3000 1350 3000 1425 3000 1500
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 9
+ 4545 915 4545 990 4545 1065 4545 1140 4545 1215 4545 1290
+ 4545 1365 4545 1440 4545 1515
2 1 0 2 0 7 50 -1 -1 0.000 0 0 7 0 0 5
- 2025 900 4425 900 4425 1500 1200 1500 1200 1050
+ 2025 900 6300 900 6300 1500 1200 1500 1200 1050
4 0 0 50 -1 16 18 0.0000 4 255 930 1050 1005 request\001
4 0 0 50 -1 16 18 0.0000 4 270 1530 1350 1305 ptlrpc_body\001
+4 0 0 50 -1 16 18 0.0000 4 270 1485 4680 1305 lustre_capa\001
4 0 0 50 -1 16 18 0.0000 4 270 1800 1050 675 OST_PUNCH\001
4 0 0 50 -1 16 18 0.0000 4 270 1200 3105 1320 ost_body\001
Lustre is a POSIX compliant file system that provides namespace and
data storage services to clients. It implements the normal file system
-functionality including creating, writing, reading, and
-removing files and directories. These file system operations are
-implemented via <<lustre-rpcs,Lustre RPCs>>, which carry
-out communication and coordination with the servers. In this section
-we present the sequence of Lustre Operations, along with their
-effects, of a variety of file system operations.
+functionality including creating, writing, reading, and removing files
+and directories. These file system operations are implemented via
+<<lustre-rpcs,Lustre RPCs>>, which carry out communication and
+coordination with the servers. In this section we present the sequence
+of Lustre RPCs for a variety of file system operations, along with
+their effects.
include::mount.txt[]
include::umount.txt[]
+//////////////////////////////////////////////////////////////////////
include::create.txt[]
+//////////////////////////////////////////////////////////////////////
include::getattr.txt[]
--------------------------------------------------------------
//////////////////////////////////////////////////////////////////////
+See <<ldlm-enqueue-rpc>>.
+
The ldlm_request structure signals that it has an intent ('lock_flags' =
LDLM_FL_HAS_INTENT), The lock descriptor's resource type is
-'loc_desc'->'l_resource'->'lr_type'=LDLM_IBITS.
+'lr_type'=LDLM_IBITS (See <<struct-ldlm-resource-desc>>).
The 'ldlm_intent' opcode is for 'getattr' (0x1000).
------------------------------
//////////////////////////////////////////////////////////////////////
+See <<ldlm-enqueue-rpc>>.
+
*4 - The OST invokes a glimps lock callback on Client2.*
Client2 previously had a lock on the desired resource, and the glimpse
------------------------------
//////////////////////////////////////////////////////////////////////
+See <<ldlm-gl-callback-rpc>>.
+
*5 - Client2 replies with LVB data for the OST.*
The OST is waiting to hear back from Client2 to update size and time
~~~~~~~~
The 'getxattr' VFS method queries extended attribute information for a
-resource given a path ('getxattr()' system call) or FID ('fgetxattr()'
-system call). If provided a path Lustre will go through a path lookup
-first (not shown [fixme: this should be a cross reference]) in order
-to determine the FID for the resource.
+resource.
Lustre maintains extended attribute information on the MDS, and a
single RPC (request and reply) retrieves the information from the
-------------------------------------------------------
//////////////////////////////////////////////////////////////////////
+See <<ldlm-enqueue.txt>>.
+
*2 - The MDT replies with an LDLM_ENQUEUE with the extended
attributes data.*
and the protocols used to implement them.
[glossary]
-back end file system::
-Storage on a server in support of the object store or of metadata is
-stored on the back end file system. The two available file systems are
-currently ldiskfs and ZFS.
+Object Storage Device (OSD)::
+The OSD is the storage on a server that holds objects or metadata. The
+two kinds of file system available ofr OSDs are ldiskfs and ZFS.
Client::
A Lustre client is a computer taking advantage of the services
timeout value without otherwise indicating the request has progressed.
export::
-The term for the per-client shared state information held by a server.
+The term for the per-client shared state information held by a
+server. See <<struct-obd-export>>.
extent:: A range of offsets in a file.
import::
-The term for the per-target shared state information held by a client.
+The term for the per-target shared state information held by a
+client. See <<struct_obd_import>>.
Inodes and Extended Attributes::
Metadata about a Lustre file is encoded in the extended attributes of
-an inode in the back-end file system on the MDT. Some of that
-information establishes the stripe layout of the file. The size of the
-stripe layout information may be different for each file. The amount
-of space reserved for layout information is a small as possible given
-the maximum stripe count on the file system. Clients, servers,
-and the distributed lock manager will all need to be aware of this
-size, which is communicated in the 'ocd_max_easize' filed of the
-<<obd_connect_data>> structure.
+an inode on the MDT. Some of that information establishes the stripe
+layout of the file. The size of the stripe layout information may be
+different for each file. The amount of space reserved for layout
+information is a small as possible given the maximum stripe count on
+the file system. Clients, servers, and the distributed lock manager
+will all need to be aware of this size, which is communicated in the
+'ocd_max_easize' fieled of the <<struct-obd-connect-data>> structure.
LNet::
A lower level protocol employed by PtlRPC to abstract the mechanisms
Introduction
------------
-[NOTE]
-I am leaving the introductory content here for now, but it is the last
-thing that should be written. The following is just a very early
-sketch, and will be revised entirely once the rest of the content has
-begun to shape up.
-
The Lustre parallel file system provides a global POSIX namespace for
the computing resources of a data center. Lustre runs on Linux-based
hosts via kernel modules, and delegates block storage management to
-the back-end servers while providing object-based storage to its
+the servers while providing object-based storage to its
clients. Servers are responsible for both data objects (the contents
of actual files) and index objects (for directory information). Data
objects are gathered on Object Storage Servers (OSSs), and index
-objects are stored on Metadata Servers (MDSs). Each back-end
-storage volume is a target with Object Storage Targets (OSTs) on OSSs,
-and Metadata Targets (MDTs) on MDSs. Clients assemble the
-data from the MDTs and OSTs to present a single coherent
-POSIX-compliant file system. The clients and servers communicate and
-coordinate among themselves via network protocols. A low-level
-protocol, LNet, abstracts the details of the underlying networking
-hardware and presents a uniform interface, originally based on Sandia
-Portals <<PORTALS>>, to Lustre clients and servers. Lustre, in turn,
-layers its own protocol atop LNet. This document describes the Lustre
-protocol.
-
-Lustre runs across multiple hosts, coordinating the activities among
-those hosts via the exchange of messages over a network. On each host,
-Lustre is implemented via a collection of Linux processes (often
-called "threads"). This discussion will refer to a more formalized
-notion of 'processes' that abstract some of the thread-level
-details. The description of the activities on each host comprise a
-collection of 'abstract processes'. Each abstract process may be
-thought of as a state machine, or automaton, following a fixed set of
-rules for how it consumes messages, changes state, and produces other
-messages. We speak of the 'behavior' of a process as shorthand for the
-management of its state and the rules governing what messages it can
-consume and produce. Processes communicate with each other via
-messages. The Lustre protocol is the collection of messages the
-processes exchange along with the rules governing the behavior of
-those processes.
-
-include::transno.txt[]
+objects are stored on Metadata Servers (MDSs). Each storage volume is
+a target with Object Storage Targets (OSTs) on OSSs, and Metadata
+Targets (MDTs) on MDSs. Clients assemble the data from the MDTs and
+OSTs to present a single coherent POSIX-compliant file system. The
+clients and servers communicate and coordinate among themselves via
+network protocols. A low-level protocol, LNet, abstracts the details
+of the underlying networking hardware and presents a uniform
+interface, originally based on Sandia Portals <<PORTALS>>, to Lustre
+clients and servers. Lustre, in turn, layers its own protocol atop
+LNet. This document describes the Lustre protocol.
+
+The remainder of the introduciton presents several concepts that
+illuminate the operation of the Lustre protocol. In
+<<file-system-operations>> a subsection is devoted to each of several
+semantic operations (setattr, statfs, ...). That discussion introduces
+the RPCs of the Lustre protocol, to give an idea of how RPCs are used
+to implement the file system. In <<lustre-rpcs>> each RPC of the
+Lustre protocol is presented in turn.
+
+include::client.txt[]
+
+include::target.txt[]
+
+include::rpc.txt[]
include::connection.txt[]
+include::transno.txt[]
+
include::path_lookup.txt[]
include::lov_index.txt[]
An RPC that assists with getting a lock back from an entity that has
it.
-.LDLM_BL_CALLBACK Generic Packet Structure
-image::ldlm-bl-callback-generic.png["LDLM_BL_CALLBACK Generic Packet Structure",height=100]
+.LDLM_BL_CALLBACK Request Packet Structure
+image::ldlm-bl-callback-request.png["LDLM_BL_CALLBACK Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The ldlm-bl-callback-generic.png diagram resembles this text
+The ldlm-bl-callback-request.png diagram resembles this text
art:
LDLM_BL_CALLBACK:
--request---------------------
| ptlrpc_body | ldlm_request |
------------------------------
- --reply--------
- | ptlrpc_body |
- ---------------
//////////////////////////////////////////////////////////////////////
The request RPC resembles the simplest LDLM_ENQUEUE RPC, but only
identifies the relevant resource that the destination entity already
had a lock on. It notifies the recipient that the lock should be
-returned once and dirty write data has been flushed. The Reply is
-just an acknowledgment of receipt, and doesn't carry any further
-information about the lock or the resource.
+returned once and dirty write data has been flushed.
'ptlrpc_body'::
RPC descriptor. <<struct-ptlrpc-body>>
Description of the lock being requested. Which resource is the target,
what lock is current, and what lock desired. <<struct-ldlm-request>>
+.LDLM_BL_CALLBACK Reply Packet Structure
+image::ldlm-bl-callback-reply.png["LDLM_BL_CALLBACK Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ldlm-bl-callback-reply.png diagram resembles this text
+art:
+
+ LDLM_BL_CALLBACK:
+ --reply--------
+ | ptlrpc_body |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
+The Reply is just an acknowledgment of receipt, and doesn't carry any
+further information about the lock or the resource.
+
+'ptlrpc_body'::
+RPC descriptor. <<struct-ptlrpc-body>>
+
An RPC that notifies an entity that a requested lock is no longer needed.
-.LDLM_CANCEL Generic Packet Structure
-image::ldlm-cancel-generic.png["LDLM_CANCEL Generic Packet Structure",height=100]
+.LDLM_CANCEL Request Packet Structure
+image::ldlm-cancel-request.png["LDLM_CANCEL Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The ldlm-cancel-generic.png diagram resembles this text
+The ldlm-cancel-request.png diagram resembles this text
art:
LDLM_CANCEL:
--request---------------------
| ptlrpc_body | ldlm_request |
------------------------------
- --reply--------
- | ptlrpc_body |
- ---------------
//////////////////////////////////////////////////////////////////////
'ptlrpc_body'::
'lock_handle' field is used. The rest of the 'ldlm_request' sturcture
is not used. <<struct-ldlm-request>>
+.LDLM_CANCEL Reply Packet Structure
+image::ldlm-cancel-reply.png["LDLM_CANCEL Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ldlm-cancel-reply.png diagram resembles this text
+art:
+
+ LDLM_CANCEL:
+ --reply--------
+ | ptlrpc_body |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. <<struct-ptlrpc-body>>
An RPC that notifies an entity that a requested lock is now available.
-.LDLM_CP_CALLBACK Generic Packet Structure
-image::ldlm-cp-callback-generic.png["LDLM_CP_CALLBACK Generic Packet Structure",height=100]
+.LDLM_CP_CALLBACK Request Packet Structure
+image::ldlm-cp-callback-request.png["LDLM_CP_CALLBACK Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The ldlm-cp-callback-generic.png diagram resembles this text
+The ldlm-cp-callback-request.png diagram resembles this text
art:
LDLM_CP_CALLBACK:
--request-------------------------------
| ptlrpc_body | ldlm_request | ost_lvb |
----------------------------------------
- --reply--------
- | ptlrpc_body |
- ---------------
//////////////////////////////////////////////////////////////////////
The request RPC identifies the relevant resource that already had a
request pending. It notifies the original requester that it now has
the stated lock, along with attribute information the about the
-resource. The reply is just an acknowledgment of receipt, and doesn't
-carry any further information about the lock or the resource.
+resource.
'ptlrpc_body'::
RPC descriptor <<struct-ptlrpc-body>>.
'ost_lvb'::
Attribute data associated with a resource on an OST. <<struct-ost-lvb>>
+.LDLM_CP_CALLBACK Reply Packet Structure
+image::ldlm-cp-callback-reply.png["LDLM_CP_CALLBACK Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ldlm-cp-callback-reply.png diagram resembles this text
+art:
+
+ LDLM_CP_CALLBACK:
+ --reply--------
+ | ptlrpc_body |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
+The reply is just an acknowledgment of receipt, and doesn't
+carry any further information about the lock or the resource.
+
+'ptlrpc_body'::
+RPC descriptor <<struct-ptlrpc-body>>.
+
clients. In its simplest form it just exchanges 'ldlm_request' and
'ldlm_reply' descriptors of the desired and granted locks.
-.LDLM_ENQUEUE Generic Packet Structure
-image::ldlm-enqueue-generic.png["LDLM_ENQUEUE Generic Packet Structure",height=100]
+However, there are many variants to this RPCs. A lock request may signal
+an intention to carry out an operation once a lock has been
+granted.
+
+.LDLM_ENQUEUE Request Packet Structure
+image::ldlm-enqueue-request.png["LDLM_ENQUEUE Request Packet Structure",height=75]
//////////////////////////////////////////////////////////////////////
-The ldlm-enqueue-generic.png diagram resembles this text
+The ldlm-enqueue-request.png diagram resembles this text
art:
LDLM_ENQUEUE:
--request---------------------
| ptlrpc_body | ldlm_request |
------------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+include::struct_ldlm_request.txt[]
+
+.LDLM_ENQUEUE Reply Packet Structure
+image::ldlm-enqueue-reply.png["LDLM_ENQUEUE Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ldlm-enqueue-reply.png diagram resembles this text
+art:
+
+ LDLM_ENQUEUE:
--reply---------------------
| ptlrpc_body | ldlm_reply |
----------------------------
//////////////////////////////////////////////////////////////////////
-However, there are many variants to this RPCs. A lock request may signal
-an intention to carry out an operation once a lock has been
-granted. In the following example, the RPCs are fostering the 'intent'
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+include::struct_ldlm_reply.txt[]
+
+****
+In the following example, the RPCs are fostering the 'intent'
to look at how a resource is mapped to the available targets, so
called 'layout' information.
+****
-.LDLM_ENQUEUE Intent:Layout Generic Packet Structure
-image::ldlm-enqueue-intent-layout-generic.png["LDLM_ENQUEUE Intent:Layout Generic Packet Structure",height=100]
+.LDLM_ENQUEUE Intent:Layout Request Packet Structure
+image::ldlm-enqueue-intent-layout-request.png["LDLM_ENQUEUE Intent:Layout Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The ldlm-enqueue-intent-layout-generic.png diagram resembles this text
+The ldlm-enqueue-intent-layout-request.png diagram resembles this text
art:
LDLM_ENQUEUE:
--intent:layout request------------------------------------
| ptlrpc_body | ldlm_request |ldlm_intent | layout_intent |
- lov_mds_md |
-----------------------------------------------------------
+ | lov_mds_md |
+ --------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'ldlm_request'::
+The LDLM Request structure. See <<struct-ldlm-request>>.
+
+include::struct_ldlm_intent.txt[]
+
+include::struct_layout_intent.txt[]
+
+include::struct_lov_mds_md.txt[]
+
+.LDLM_ENQUEUE Intent:Layout Reply Packet Structure
+image::ldlm-enqueue-intent-layout-reply.png["LDLM_ENQUEUE Intent:Layout Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ldlm-enqueue-intent-layout-reply.png diagram resembles this text
+art:
+
+ LDLM_ENQUEUE:
--intent:layout reply--------------------
| ptlrpc_body | ldlm_reply | lov_mds_md |
-----------------------------------------
//////////////////////////////////////////////////////////////////////
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'ldlm_reply'::
+The LDLM Reply structure. See <<struct-ldlm-reply>>.
+
+****
+Lock Value Block::
+A Lock Value Block (LVB) is included in the LDLM_ENQUEUE reply message
+when one of three things needs to be communicated back to the
+requester. The three alternatives are captured by the
+'ost_lvb' structure, the 'lov_mds_md' structure, and one other related
+to quotas (and not yet incorporated into this document). LDLM_ENQUEUE
+reply RPCs may include a zero length instance of an LVB.
+****
+
+'lov_mds_md'::
+The LOV MDS MD structure. See <<struct-lov-mds-md>>.
+
And in this example the intent is to get attribute information.
-.LDLM_ENQUEUE Intent:Getattr Generic Packet Structure
-image::ldlm-enqueue-intent-getattr-generic.png["LDLM_ENQUEUE Intent:Getattr Generic Packet Structure",height=150]
+.LDLM_ENQUEUE Intent:Getattr Request Packet Structure
+image::ldlm-enqueue-intent-getattr-request.png["LDLM_ENQUEUE Intent:Getattr Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The ldlm-enqueue-intent-getattr-generic.png diagram resembles this text
+The ldlm-enqueue-intent-getattr-request.png diagram resembles this text
art:
LDLM_ENQUEUE:
--intent:getattr request-------------------------------
| ptlrpc_body | ldlm_request | ldlm_intent | mdt_body |
- lustre_capa |name |
-------------------------------------------------------
+ | lustre_capa | name |
+ ----------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'ldlm_request'::
+The LDLM Request structure. See <<struct-ldlm-request>>.
+
+'ldlm_intent'::
+The LDLM Intent structure. See <<struct-ldlm-intent>>.
+
+'mdt_body'::
+Metadata about the resource. See <<struct-mdt-body>>.
+
+'lustre_capa'::
+A "capabilities" structure. See <<struct-lustre-capa>>.
+
+'name'::
+A text field supplying the name of the desired resource.
+
+.LDLM_ENQUEUE Intent:Getattr Reply Packet Structure
+image::ldlm-enqueue-intent-getattr-reply.png["LDLM_ENQUEUE Intent:Getattr Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ldlm-enqueue-intent-getattr-reply.png diagram resembles this text
+art:
+
+ LDLM_ENQUEUE:
--intent:getattr reply--------------------------
| ptlrpc_body | ldlm_reply | mdt_body | mdt_md |
------------------------------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'ldlm_reply'::
+The LDLM Reply structure. See <<struct-ldlm-reply>>.
+
+'mdt_body'::
+Metadata about the resource. See <<struct-mdt-body>>.
+
+'mdt_md'::
+Layout data for the resource. This buffer is optional and will appear
+as zero length in some packets.
+
+.LDLM_ENQUEUE Intent:Lvb Reply Packet Structure
+image::ldlm-enqueue-intent-lvb-reply.png["LDLM_ENQUEUE Intent:Lvb Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ldlm-enqueue-intent-lvb-reply.png diagram resembles this text
+art:
+
+ LDLM_ENQUEUE:
--intent:lvb reply--------------------
| ptlrpc_body | ldlm_reply | ost_lvb |
--------------------------------------
//////////////////////////////////////////////////////////////////////
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'ldlm_reply'::
+The LDLM Reply structure. See <<struct-ldlm-reply>>.
+
+include::struct_ost_lvb.txt[]
+
Here is another example of an intent, in this case the 'getxattr' intent.
-.LDLM_ENQUEUE Intent:Getxattr Generic Packet Structure
-image::ldlm-enqueue-intent-getxattr-generic.png["LDLM_ENQUEUE Intent:Getxattr Generic Packet Structure",height=125]
+.LDLM_ENQUEUE Intent:Getxattr Request Packet Structure
+image::ldlm-enqueue-intent-getxattr-request.png["LDLM_ENQUEUE Intent:Getxattr Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The ldlm-enqueue-intent-getxattr-generic.png diagram resembles this text
+The ldlm-enqueue-intent-getxattr-request.png diagram resembles this text
art:
LDLM_ENQUEUE:
- --intent:getxattr request------------------------------------
- | ptlrpc_body | ldlm_request |ldlm_intent | mdt_body | capa |
- -------------------------------------------------------------
- --intent:getxattr reply----------------------------------------
- | ptlrpc_body | ldlm_reply | mdt_body | lov_mds_md | ACL data |
- | EA data | EA vals | EA lens |
- ---------------------------------------------------------------
+ --intent:getxattr request-----------------------------
+ | ptlrpc_body | ldlm_request |ldlm_intent | mdt_body |
+ ------------------------------------------------------
+ | lustre_capa |
+ ---------------
//////////////////////////////////////////////////////////////////////
'ptlrpc_body'::
-RPC descriptor <<struct-ptlrpc-body>>.
+RPC descriptor. See <<struct-ptlrpc-body>>.
-include::struct_ldlm_request.txt[]
+'ldlm_request'::
+The LDLM Request structure. See <<struct-ldlm-request>>.
-include::struct_ldlm_intent.txt[]
-
-include::struct_layout_intent.txt[]
+'ldlm_intent'::
+The LDLM Intent structure. See <<struct-ldlm-intent>>.
'mdt_body'::
-In a request, an indication (in the 'mbo_valid' field) of what
-attributes the requester would like. In a reply, (again based on
-'mbo_valid') the values being returned. <<struct-mdt-body>>
+Metadata about the resource. See <<struct-mdt-body>>.
'lustre_capa'::
-So called "capabilities" structure. This is deprecated in recent
-versions of Lustre, and commonly appears in the packet header as a zero
-length buffer.
+A "capabilities" structure. See <<struct-lustre-capa>>.
-'name'::
-A text field supplying the name of the desired resource.
+.LDLM_ENQUEUE Intent:Getxattr Reply Packet Structure
+image::ldlm-enqueue-intent-getxattr-reply.png["LDLM_ENQUEUE Intent:Getxattr Reply Packet Structure",height=50]
-include::struct_ldlm_reply.txt[]
+//////////////////////////////////////////////////////////////////////
+The ldlm-enqueue-intent-getxattr-reply.png diagram resembles this text
+art:
+
+ LDLM_ENQUEUE:
+ --intent:getxattr reply----------------------------------------
+ | ptlrpc_body | ldlm_reply | mdt_body | lov_mds_md | ACL data |
+ | EA data | EA vals | EA lens |
+ ---------------------------------------------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'ldlm_reply'::
+The LDLM Reply structure. See <<struct-ldlm-reply>>.
+
+'mdt_body'::
+Metadata about the resource. See <<struct-mdt-body>>.
+
+'lov_mds_md'::
+The LOV MDS MD structure. See <<struct-lov-mds-md>>.
'ACL data'::
Access Control List data associated with a resource.
attribute in the "EA vals" block of data. Thus the sum of those sizes
equals the length of the "EA vals".
-Lock Value Block::
-A Lock Value Block (LVB) is included in the LDLM_ENQUEUE reply message
-when one of three things needs to be communicated back to the
-requester. The three alternatives are captured by the
-'ost_lvb' structure, the 'lov_mds_md' structure, and one other related
-to quotas (and not yet incorporated into this document). LDLM_ENQUEUE
-reply RPCs may include a zero length instance of an LVB.
-
-include::struct_ost_lvb.txt[]
-
-include::struct_lov_mds_md.txt[]
-
An RPC that assists with getting a lock back from an entity that has
it.
-.LDLM_GL_CALLBACK Generic Packet Structure
-image::ldlm-gl-callback-generic.png["LDLM_GL_CALLBACK Generic Packet Structure",height=100]
+.LDLM_GL_CALLBACK Request Packet Structure
+image::ldlm-gl-callback-request.png["LDLM_GL_CALLBACK Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The ldlm-gl-callback-generic.png diagram resemgles this text
+The ldlm-gl-callback-request.png diagram resemgles this text
art:
LDLM_GL_CALLBACK:
--request---------------------
| ptlrpc_body | ldlm_request |
------------------------------
- --reply------------------
- | ptlrpc_body | ost_lvb |
- -------------------------
//////////////////////////////////////////////////////////////////////
The request RPC resembles the simplest LDLM_ENQUEUE RPC, but only
identifies the relevant resource that the destination entity already
had a lock on. It asks the recipient to flush its dirty write cache,
and notify the requester of size and time attributes once that is
-done. The reply updates the attributes on the requester.
+done.
'ptlrpc_body'::
RPC descriptor. <<struct-ptlrpc-body>>
Description of the lock being requested. Which resource is the target,
what lock is current, and what lock desired. <<struct-ldlm-request>>
+.LDLM_GL_CALLBACK Reply Packet Structure
+image::ldlm-gl-callback-reply.png["LDLM_GL_CALLBACK Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ldlm-gl-callback-reply.png diagram resemgles this text
+art:
+
+ LDLM_GL_CALLBACK:
+ --reply------------------
+ | ptlrpc_body | ost_lvb |
+ -------------------------
+//////////////////////////////////////////////////////////////////////
+
+The reply updates the attributes on the requester.
+
+'ptlrpc_body'::
+RPC descriptor. <<struct-ptlrpc-body>>
+
'ost_lvb'::
Attribute data associated with a resource on an OST. <<struct-ost-lvb>>
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[[llog-origin-handle-create-rpc]]
-.LLOG_ORIGIN_HANDLE_CREATE Generic Packet Structure
-image::llog-origin-handle-create-generic.png["LLOG_ORIGIN_HANDLE_CREATE Generic Packet Structure",height=100]
+Open a log object.
+
+.LLOG_ORIGIN_HANDLE_CREATE Request Packet Structure
+image::llog-origin-handle-create-request.png["LLOG_ORIGIN_HANDLE_CREATE Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The llog-origin-handle-create-generic.png diagram resembles this text
+The llog-origin-handle-create-request.png diagram resembles this text
art:
LLOG_ORIGIN_HANDLE_CREATE:
--request----------------------------
| ptlrpc_body | llogd_body | string |
-------------------------------------
- --reply---------------------
- | ptlrpc_body | llogd_body |
- ----------------------------
//////////////////////////////////////////////////////////////////////
'ptlrpc_body':: RPC descriptor. See <<struct-ptlrpc-body>>.
include::struct_llogd_body.txt[]
'string':: The name of the log.
+
+.LLOG_ORIGIN_HANDLE_CREATE Reply Packet Structure
+image::llog-origin-handle-create-reply.png["LLOG_ORIGIN_HANDLE_CREATE Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The llog-origin-handle-create-reply.png diagram resembles this text
+art:
+
+ LLOG_ORIGIN_HANDLE_CREATE:
+ --reply---------------------
+ | ptlrpc_body | llogd_body |
+ ----------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body':: RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'llogd_body'::
+The 'llogd_body' structure. See <<struct-llogd-body>>.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[[llog-origin-handle-next-block-rpc]]
-.LLOG_ORIGIN_HANDLE_NEXT_BLOCK Generic Packet Structure
-image::llog-origin-handle-next-block-generic.png["LLOG_ORIGIN_HANDLE_NEXT_BLOCK Generic Packet Structure",height=100]
+Read information from a log object.
+
+.LLOG_ORIGIN_HANDLE_NEXT_BLOCK Request Packet Structure
+image::llog-origin-handle-next-block-request.png["LLOG_ORIGIN_HANDLE_NEXT_BLOCK Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The llog-origin-handle-next-block-generic.png diagram resembles this
+The llog-origin-handle-next-block-request.png diagram resembles this
text art:
LLOG_ORIGIN_HANDLE_NEXT_BLOCK:
--request-------------------
| ptlrpc_body | llogd_body |
----------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body':: RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'llogd_body':: See <<struct-llogd-body>>
+
+.LLOG_ORIGIN_HANDLE_NEXT_BLOCK Reply Packet Structure
+image::llog-origin-handle-next-block-reply.png["LLOG_ORIGIN_HANDLE_NEXT_BLOCK Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The llog-origin-handle-next-block-reply.png diagram resembles this
+text art:
+
+ LLOG_ORIGIN_HANDLE_NEXT_BLOCK:
--reply------------------------------
| ptlrpc_body | llogd_body | eadata |
-------------------------------------
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[[llog-origin-handle-read-header-rpc]]
-.LLOG_ORIGIN_HANDLE_READ_HEADER Generic Packet Structure
-image::llog-origin-handle-read-header-generic.png["LLOG_ORIGIN_HANDLE_READ_HEADER Generic Packet Structure",height=100]
+Read the header for a log object.
+
+.LLOG_ORIGIN_HANDLE_READ_HEADER Request Packet Structure
+image::llog-origin-handle-read-header-request.png["LLOG_ORIGIN_HANDLE_READ_HEADER Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The llog-origin-handle-read-header-generic.png diagram resembles this
+The llog-origin-handle-read-header-request.png diagram resembles this
text art:
LLOG_ORIGIN_HANDLE_READ_HEADER:
--request-------------------
| ptlrpc_body | llogd_body |
----------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body':: RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'llogd_body':: See <<struct-llogd-body>>
+
+.LLOG_ORIGIN_HANDLE_READ_HEADER Reply Packet Structure
+image::llog-origin-handle-read-header-reply.png["LLOG_ORIGIN_HANDLE_READ_HEADER Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The llog-origin-handle-read-header-reply.png diagram resembles this
+text art:
+
+ LLOG_ORIGIN_HANDLE_READ_HEADER:
--reply--------------------------
| ptlrpc_body | llog_log_header |
---------------------------------
replies unless the message encountered a fatal error before it could
be processed, in which case it will contain PTLRPC_MSG_ERR.
-The type of operation requested is denoted by the 'pb_opc' op-code field
-of the RPC request. Note that as a general matter, the receipt by a
-client of the reply message only assures the client that the server
-has initiated the action, if any, but does not guarantee that any
-modification has been committed to persistent storage. See the
-discussion on <<transno,transaction numbers>> and <<recovery>> for
-how the client is given confirmation that a request has been completed.
+The type of operation requested is denoted by the 'pb_opc' op-code
+field of the RPC request. Note that as a general matter, the receipt
+by a client of the reply message only assures the client that the
+server has initiated the action, if any, but does not guarantee that
+any modification has been committed to persistent storage. See the
+discussion in <<transno>> and <<recovery>> for how the client is given
+confirmation that a request has been completed.
include::ost_setattr.txt[]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[[mds-connect-rpc]]
-.MDS_CONNECT Generic Packet Structure
-image::mds-connect-generic.png["MDS_CONNECT Generic Packet Structure",height=100]
+The general behavior of MDS_CONNECT RPCs closely resembles that of
+OST_CONNECT RPCs (See <<ost-connect-rpc>>). The actual RPC's 'pb_opc'
+value will be different as are the names of the targets and the
+specific values in the 'obd_connect_data' structure.
+
+.MDS_CONNECT Request Packet Structure
+image::mds-connect-request.png["MDS_CONNECT Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The mds-connect-generic.png diagram resembles this text art:
+The mds-connect-request.png diagram resembles this text art:
MDS_CONNECT:
- --request--------------------------------------------
- | ptlrpc_body | obd_uuid | obd_uuid | lustre_handle |
- -----------------------------------------------------
- | obd_connect_data |
+ --request--------------------------------------------------
+ | ptlrpc_body | target_uuid | client_uuid | lustre_handle |
+ -----------------------------------------------------------
+ | obd_connect_data |
---------------------
- --reply---------------------------
- | ptlrpc_body | obd_connect_data |
- ----------------------------------
//////////////////////////////////////////////////////////////////////
'ptlrpc_body'::
-RPC descriptor.
+RPC descriptor. See <<struct-ptlrpc-body>>. In a connect message the
+'ptlrpc_body' field 'pb_handle', which is a <<struct-lustre-handle>>,
+is 0. That 'lustre_handle' is distinct from the one mentioned
+below.
+
+'target_uuid'::
+A string identifying the target. See <<struct-obd-uuid>>. The client
+learns the UUID strings for the targets via an interaction with the
+MGS.
-'obd_uuid'::
-UUIDs of the target (first) and client (second) entities. See
-<<struct-obd-uuid>>.
+'client_uuid'::
+A string with the client's own UUID. This is also a
+<<struct-obd-uuid>>. The target sets the 'exp_client_uuid' field of
+its 'eport' structure to this value.
'lustre_handle'::
-See <<struct-lustre-handle>>.
+See <<struct-lustre-handle>>. This 'lustre_handle' is distinct from
+the 'pb_handle' field in the 'ptlrpc_body'. This 'lustre_handle'
+provied a unique 'cookie' to identify this client for this connection
+attempt. After a disconnect, a subsequent connect RPC will have a
+different value. The target preserves this cookie in the 'exp_handle'
+field of its 'obd_export' structure for this client. In that way it
+can distinguish between a resent connection request and an entirely
+new connection request.
'obd_connect_data'::
See <<struct-obd-connect-data>>.
-The 'ocd_connect_flags' buffer is initialized to the set of features
-that the client supports for metadata targets. For Lustre 2.7 clients
-these features are OBD_CONNECT_IBITS, OBD_CONNECT_NODEVOH,
-OBD_CONNECT_ATTRFID, OBD_CONNECT_VERSION, OBD_CONNECT_BRW_SIZE,
-OBD_CONNECT_CANCELSET, OBD_CONNECT_FID, OBD_CONNECT_AT, OBD_CONNECT_LOV_V3,
-OBD_CONNECT_VBR, OBD_CONNECT_FULL20, OBD_CONNECT_64BITHASH,
-OBD_CONNECT_EINPROGRESS, OBD_CONNECT_JOBSTATS, OBD_CONNECT_LVB_TYPE,
-OBD_CONNECT_LAYOUTLOCK, OBD_CONNECT_PINGLESS, OBD_CONNECT_MAX_EASIZE,
-OBD_CONNECT_FLOCK_DEAD, OBD_CONNECT_DISP_STRIPE, OBD_CONNECT_LFSCK, and
-OBD_CONNECT_OPEN_BY_FID, as described in <<obd-connect-flags>>.
-
The 'ocd_brw_size' field is set to the largest size in bytes that the
-client can use for bulk transfers. The 'ocd_ibits_known' field is set to
-the supported set of IBITS locks, as of Lustre 2.7 these are
-MDS_INODELOCK_LOOKUP, MDS_INODELOCK_UPDATE, MDS_INODELOCK_OPEN,
-MDS_INODELOCK_LAYOUT, MDS_INODELOCK_PERM, and MDS_INODELOCK_XATTR,
-as described in <<mds-inode-bits-locks>>.
+client can use for bulk transfers.
+
+The 'ocd_ibits_known' field is set to the supported set of IBITS
+locks. As of Lustre 2.7 these are MDS_INODELOCK_LOOKUP,
+MDS_INODELOCK_UPDATE, MDS_INODELOCK_OPEN, MDS_INODELOCK_LAYOUT,
+MDS_INODELOCK_PERM, and MDS_INODELOCK_XATTR>.
The 'ocd_version' field contains the version of Lustre
running on the client. Other fields in the 'obd_connect_data'
structures are zero.
-Once the server receives the 'obd_connect_client' message on behalf of
-the given target it replies with an 'obd_connect_server' message. In
-the reply message the server sets the 'pb_handle' to uniquely
-identify this connection for subsequent communication. The client notes
-that handle in its import for the given target. The reply also returns
-the 'obd_connect_data' as interpreted by the MDS. Any features that
-the MDS does not support are masked out of 'ocd_feature_flags'. Supported
-features that have assigned fields are filled in by the MDS.
-
-fixme: Are there circumstances that could lead to the 'status'
-value in the reply being non-zero? What would lead to that and what
-error values would result?
-ans: yes, at least the standard errors
-
-The target maintains the last committed transaction for a client in
-its export for that client. If this is the first connection, then that
-last transaction value would just be zero. If there were previous
-transactions for the client, then the transaction number for the last
-such committed transaction is put in the <<struct-ptlrpc-body>>
-'pb_last_committed' field.
-
-In a connection request the operation is not file system modifying, so
-the 'pb_transno' value will be zero in the reply as well.
-
-fixme: there is still some work to be done about how the fields are
-managed.
+.MDS_CONNECT Reply Packet Structure
+image::mds-connect-reply.png["MDS_CONNECT Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-connect-reply.png diagram resembles this text art:
+
+ MDS_CONNECT:
+ --reply---------------------------
+ | ptlrpc_body | obd_connect_data |
+ ----------------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>. The 'pb_handle' field has
+the value, $C_t$, uniquely identifying that target. The value $C_t$ is
+preserved in the 'imp_remote_handle' field of the client's import
+structure for this target. See <<struct-obd-import>>.
+
+'obd_connect_data'::
+See <<struct-obd-connect-data>>.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[[mds-disconnect-rpc]]
-.MDS_DISCONNECT Generic Packet Structure
-image::mds-disconnect-generic.png["MDS_DISCONNECT Generic Packet Structure",height=100]
+Disconnect from an MDS.
+
+.MDS_DISCONNECT Request Packet Structure
+image::mds-disconnect-request.png["MDS_DISCONNECT Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The mds-disconnect-generic.png diagram resembles this text art:
+The mds-disconnect-request.png diagram resembles this text art:
MDS_DISCONNECT:
--request------
| ptlrpc_body |
---------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+.MDS_DISCONNECT Reply Packet Structure
+image::mds-disconnect-reply.png["MDS_DISCONNECT Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-disconnect-reply.png diagram resembles this text art:
+
+ MDS_DISCONNECT:
--reply--------
| ptlrpc_body |
---------------
//////////////////////////////////////////////////////////////////////
-The information exchanged in a DISCONNECT message is that normally
-conveyed in the RPC descriptor, as described in <<struct-ptlrpc-body>>.
-
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[[mds-getattr-rpc]]
-.MDS_GETATTR Generic Packet Structure
-image::mds-getattr-generic.png["MDS_GETATTR Generic Packet Structure",height=100]
+Get attribute information for a resource on an MDT.
+
+.MDS_GETATTR Request Packet Structure
+image::mds-getattr-request.png["MDS_GETATTR Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The mds-getattr-generic.png diagram resembles this text art:
+The mds-getattr-request.png diagram resembles this text art:
MDS_GETATTR:
--request-------------------------------
| ptlrpc_body | mdt_body | lustre_capa |
----------------------------------------
- --reply------------------------------------------------
- | ptlrpc_body | mdt_body | MDS_MD | ACL | lustre_capa |
- -------------------------------------------------------
- | lustre_capa |
- ---------------
//////////////////////////////////////////////////////////////////////
'ptlrpc_body'::
-RPC descriptor. See <<struct-ptlrpc_body>>.
+RPC descriptor. See <<struct-ptlrpc-body>>.
include::struct_mdt_body.txt[]
+'lustre_capa'::
+A "capabilities" structure. See <<struct-lustre-capa>>.
+
+.MDS_GETATTR Reply Packet Structure
+image::mds-getattr-reply.png["MDS_GETATTR Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-getattr-reply.png diagram resembles this text art:
+
+ MDS_GETATTR:
+ --reply----------------------------------------------
+ | ptlrpc_body | mdt_body | MDS_MD | ACL | fid1_capa |
+ -----------------------------------------------------
+ | fid2_capa |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'mdt_body'::
+Metadata about the resource. See <<struct-mdt-body>>.
+
MDS_MD::
Needs more detail.
ACL::
Needs more detail.
-'lustre_capa'::
-So called "capabilities" structure. This is deprecated in recent
-versions of Lustre, and commonly appears in the packet header as a zero
-length buffer.
+'fid1_capa'::
+The capabilities structure for the first FID in the 'mdt_body'. See
+<<struct-lustre-capa>>.
+
+'fid2_capa'::
+The capabilities structure for the second FID in the 'mdt_body'. See
+<<struct-lustre-capa>>.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[[mds-getstatus-rpc]]
-.MDS_GETSTATUS Generic Packet Structure
-image::mds-getstatus-generic.png["MDS_GETSTATUS Generic Packet Structure",height=100]
+Get the attributes of the mountpoint of the file system.
+
+.MDS_GETSTATUS Request Packet Structure
+image::mds-getstatus-request.png["MDS_GETSTATUS Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The mds-getstatus-generic.png diagram resembles this text art:
+The mds-getstatus-request.png diagram resembles this text art:
MDS_GETSTATUS:
--request-----------------
| ptlrpc_body | mdt_body |
--------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'mdt_body'::
+See <<struct-mdt-body>>.
+
+In the request message, the 'mdt_body' is normally empty.
+
+.MDS_GETSTATUS Reply Packet Structure
+image::mds-getstatus-reply.png["MDS_GETSTATUS Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-getstatus-reply.png diagram resembles this text art:
+
+ MDS_GETSTATUS:
--reply-------------------
| ptlrpc_body | mdt_body |
--------------------------
//////////////////////////////////////////////////////////////////////
'ptlrpc_body'::
-RPC descriptor. See <<struct-ptlrpc_body>>.
+RPC descriptor. See <<struct-ptlrpc-body>>.
'mdt_body'::
See <<struct-mdt-body>>.
-In the request message, the 'mdt_body' is normally empty.
-
In the reply message, the 'mdt_body' contains only the FID of the
filesystem ROOT in 'mbo_fid1'. The client can then use the returned
FID to fetch inode attributes for the root inode of the local
mountpoint.
-
An RPC that fetches extended attributes for a resource.
-.MDS_GETXATTR Generic Packet Structure
-image::mds-getxattr-generic.png["MDS_GETXATTR Generic Packet Structure",height=100]
+.MDS_GETXATTR Request Packet Structure
+image::mds-getxattr-request.png["MDS_GETXATTR Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The mds-getxattr-generic.png diagram resembles this text art:
+The mds-getxattr-request.png diagram resembles this text art:
MDS_GETXATTR:
--request-----------------------------------------------
| ptlrpc_body | mdt_body | lustre_capa | name | eadata |
--------------------------------------------------------
- --reply----------------------------
- | ptlrpc_body | mdt_body | eadata |
- -----------------------------------
//////////////////////////////////////////////////////////////////////
'ptlrpc_body'::
-RPC descriptor.
+RPC descriptor. See <<struct-ptlrpc-body>>.
'mdt_body'::
-Metadata about the resource.
+Metadata about the resource. See <<struct-mdt-body>>.
'lustre_capa'::
-So called "capabilities" structure. This is deprecated in recent
-versions of Lustre, and commonly appears in the packet header as a zero
-length buffer.
+A "capabilities" structure. See <<struct-lustre-capa>>.
'name'::
A text field supplying the name of the desired resource.
Information about extended attributes. This buffer is optional and will
appear as zero length in some packets.
+.MDS_GETXATTR Reply Packet Structure
+image::mds-getxattr-reply.png["MDS_GETXATTR Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-getxattr-reply.png diagram resembles this text art:
+
+ MDS_GETXATTR:
+ --reply----------------------------
+ | ptlrpc_body | mdt_body | eadata |
+ -----------------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'mdt_body'::
+Metadata about the resource. See <<struct-mdt-body>>.
+
+'eadata'::
+Information about extended attributes. This buffer is optional and will
+appear as zero length in some packets.
+
An RPC that implements the 'setattr' sub-command of the MDS_REINT.
-.MDS_REINT:REINT_SETATTR Generic Packet Structure
-image::mds-reint-setattr-generic.png["MDS_REINT:REINT_SETATTR Generic Packet Structure",height=100]
+.MDS_REINT:REINT_SETATTR Request Packet Structure
+image::mds-reint-setattr-request.png["MDS_REINT:REINT_SETATTR Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The mds-reint-setattr-generic.png diagram resembles this text art:
+The mds-reint-setattr-request.png diagram resembles this text art:
MDS_REINT:
--REINT_SETATTR-request-------------------------------------
| ptlrpc_body | mdt_rec_setattr | lustre_capa | mdt_ioepoc |
- eadata | llog_cookie | ldlm_request |
------------------------------------------------------------
+ | eadata | llog_cookie | ldlm_request |
+ ---------------------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+include::struct_mdt_rec_setattr.txt[]
+
+'lustre_capa'::
+A "capabilities" structure. See <<struct-lustre-capa>>.
+
+'mdt_ioepoch'::
+Identifying "epoch" information. This buffer is optional and will
+appear as zero length in some packets.
+
+'eadata'::
+Information about extended attributes. This buffer is optional and will
+appear as zero length in some packets.
+
+'llog_cookie'::
+A log handle. This buffer is optional and will appear as zero length
+in some packets.
+
+'ldlm_request'::
+A structure specifying a lock that can be the subject of early lock
+cancellation. See <<early-lock-cancellation>>.
+
+.MDS_REINT:REINT_SETATTR Reply Packet Structure
+image::mds-reint-setattr-reply.png["MDS_REINT:REINT_SETATTR Reply Packet Structure",height=50]
- --REINT_SETATTR-reply----------------------------------
- | ptlrpc_body | mdt_body | mdt_md | acl | lustre_capa |
- lustre_capa |
- -------------------------------------------------------
//////////////////////////////////////////////////////////////////////
+The mds-reint-setattr-reply.png diagram resembles this text art:
+
+ MDS_REINT:
+ --REINT_SETATTR-reply--------------------------------
+ | ptlrpc_body | mdt_body | mdt_md | acl | fid1_capa |
+ -----------------------------------------------------
+ | fid2_capa |
+ -------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'mdt_body'::
+Metadata about the resource.
+
+'mdt_md'::
+Layout data for the resource. This buffer is optional and will appear
+as zero length in some packets.
+
+'acl'::
+Access control list data. This buffer is optional and will appear as
+zero length in some packets.
+
+'fid1_capa'::
+The capabilities structure for the first FID in the 'mdt_body'. See
+<<struct-lustre-capa>>.
+
+'fid2_capa'::
+The capabilities structure for the second FID in the 'mdt_body'. See
+<<struct-lustre-capa>>.
REINT_SETXATTR RPC
^^^^^^^^^^^^^^^^^^
An RPC that implements the 'setxattr' sub-command of the MDS_REINT.
-.MDS_REINT:REINT_SETXATTR Generic Packet Structure
-image::mds-reint-setxattr-generic.png["MDS_REINT:REINT_SETXATTR Generic Packet Structure",height=100]
+.MDS_REINT:REINT_SETXATTR Request Packet Structure
+image::mds-reint-setxattr-request.png["MDS_REINT:REINT_SETXATTR Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The mds-reint-setxattr-generic.png diagram resembles this text art:
+The mds-reint-setxattr-request.png diagram resembles this text art:
MDS_REINT:
--REINT_SETXATTR-request-------------------------------------
| ptlrpc_body | mdt_rec_setxattr | lustre_capa | mdt_ioepoc |
- eadata | llog_cookie | ldlm_request |
-------------------------------------------------------------
-
- --REINT_SETXATTR-reply---------------------------------
- | ptlrpc_body | mdt_body | mdt_md | acl | lustre_capa |
- lustre_capa |
- -------------------------------------------------------
+ | eadata | llog_cookie | ldlm_request |
+ ---------------------------------------
//////////////////////////////////////////////////////////////////////
'ptlrpc_body'::
-RPC descriptor.
-
-include::struct_mdt_rec_setattr.txt[]
+RPC descriptor. See <<struct-ptlrpc-body>>.
include::struct_mdt_rec_setxattr.txt[]
'lustre_capa'::
-So called "capabilities" structure. This is deprecated in recent
-versions of Lustre, and commonly appears in the packet header as a zero
-length buffer.
+A "capabilities" structure. See <<struct-lustre-capa>>.
'mdt_ioepoch'::
Identifying "epoch" information. This buffer is optional and will
A structure specifying a lock that can be the subject of early lock
cancellation. See <<early-lock-cancellation>>.
+.MDS_REINT:REINT_SETXATTR Reply Packet Structure
+image::mds-reint-setxattr-reply.png["MDS_REINT:REINT_SETXATTR Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-reint-setxattr-reply.png diagram resembles this text art:
+
+ MDS_REINT:
+ --REINT_SETXATTR-reply---------------------------------
+ | ptlrpc_body | mdt_body | mdt_md | acl | fid1_capa |
+ -------------------------------------------------------
+ | fid2_capa |
+ -------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
'mdt_body'::
Metadata about the resource.
Access control list data. This buffer is optional and will appear as
zero length in some packets.
+'fid1_capa'::
+The capabilities structure for the first FID in the 'mdt_body'. See
+<<struct-lustre-capa>>.
+
+'fid2_capa'::
+The capabilities structure for the second FID in the 'mdt_body'. See
+<<struct-lustre-capa>>.
+
to keep its information about free space and utilization updated.
That allows the MDS to make more optimal file allocation decisions.
-.MDS_STATFS Generic Packet Structure
-image::mds-statfs-generic.png["MDS_STATFS Generic Packet Structure",height=100]
+.MDS_STATFS Request Packet Structure
+image::mds-statfs-request.png["MDS_STATFS Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The mds-statfs-generic.png diagram resembles this text art:
+The mds-statfs-request.png diagram resembles this text art:
MDS_STATFS:
--request------
| ptlrpc_body |
---------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body':: RPC descriptor. See <<struct-ptlrpc-body>>.
+
+.MDS_STATFS Reply Packet Structure
+image::mds-statfs-reply.png["MDS_STATFS Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-statfs-reply.png diagram resembles this text art:
+
+ MDS_STATFS:
--reply---------------------
| ptlrpc_body | obd_statfs |
----------------------------
//////////////////////////////////////////////////////////////////////
-'ptlrpc_body':: RPC descriptor. See <<ptlrpc_body>>.
+'ptlrpc_body':: RPC descriptor. See <<struct-ptlrpc-body>>.
'obd_statfs':: Statfs information about the target. See
-<struct-obd-statfs>>.
+<<struct-obd-statfs>>.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[[mgs-config-read-rpc]]
-.MGS_CONFIG_READ Generic Packet Structure
-image::mgs-config-read-generic.png["MGS_CONFIG_READ Generic Packet Structure",height=100]
+Read configuration information about the file system.
+
+.MGS_CONFIG_READ Request Packet Structure
+image::mgs-config-read-request.png["MGS_CONFIG_READ Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The mgs-config-read-generic.png diagram resembles this text art:
+The mgs-config-read-request.png diagram resembles this text art:
MGS_CONFIG_READ:
--request------------------------
| ptlrpc_body | mgs_config_body |
---------------------------------
- --reply--------------------------
- | ptlrpc_body | mgs_config_body |
- ---------------------------------
//////////////////////////////////////////////////////////////////////
'ptlrpc_body':: RPC descriptor. See <<struct-ptlrpc-body>>.
-include::struct_mgs_config_body.txt
+include::struct_mgs_config_body.txt[]
+
+.MGS_CONFIG_READ Reply Packet Structure
+image::mgs-config-read-reply.png["MGS_CONFIG_READ Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mgs-config-read-reply.png diagram resembles this text art:
+
+ MGS_CONFIG_READ:
+ --reply-------------------------
+ | ptlrpc_body | mgs_config_res |
+ --------------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body':: RPC descriptor. See <<struct-ptlrpc-body>>.
+
+include::struct_mgs_config_res.txt[]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[[mgs-connect-rpc]]
-.MGS_CONNECT Generic Packet Structure
-image::mgs-connect-generic.png["MGS_CONNECT Generic Packet Structure",height=100]
+The general behavior of MGS_CONNECT RPCs closely resembles that of
+OST_CONNECT RPCs (See <<ost-connect-rpc>>) and MDS_CONNECt RPCs. The
+actual RPC's 'pb_opc' value will be different as are the names of the
+targets and the specific values in the 'obd_connect_data' structure.
+
+.MGS_CONNECT Request Packet Structure
+image::mgs-connect-request.png["MGS_CONNECT Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The mgs-connect-generic.png diagram resembles this text art:
+The mgs-connect-request.png diagram resembles this text art:
MGS_CONNECT:
- --request--------------------------------------------
- | ptlrpc_body | obd_uuid | obd_uuid | lustre_handle |
- -----------------------------------------------------
+ --request--------------------------------------------------
+ | ptlrpc_body | target_uuid | client_uuid | lustre_handle |
+ -----------------------------------------------------------
| obd_connect_data |
---------------------
- --reply---------------------------
- | ptlrpc_body | obd_connect_data |
- ----------------------------------
//////////////////////////////////////////////////////////////////////
'ptlrpc_body'::
-RPC descriptor.
+RPC descriptor. See <<struct-ptlrpc-body>>. In a connect message the
+'ptlrpc_body' field 'pb_handle', which is a <<struct-lustre-handle>>,
+is 0. That 'lustre_handle' is distinct from the one mentioned
+below.
-'obd_uuid'::
-UUIDs of the target (first) and client (second) entities. See
-<<struct-obd-uuid>>.
+'target_uuid'::
+A string identifying the target. See <<struct-obd-uuid>>. The client
+learns the UUID strings for the targets via an interaction with the
+MGS.
+
+'client_uuid'::
+A string with the client's own UUID. This is also a
+<<struct-obd-uuid>>. The target sets the 'exp_client_uuid' field of
+its 'eport' structure to this value.
'lustre_handle'::
-See <<struct-lustre-handle>>.
+See <<struct-lustre-handle>>. This 'lustre_handle' is distinct from
+the 'pb_handle' field in the 'ptlrpc_body'. This 'lustre_handle'
+provied a unique 'cookie' to identify this client for this connection
+attempt. After a disconnect, a subsequent connect RPC will have a
+different value. The target preserves this cookie in the 'exp_handle'
+field of its 'obd_export' structure for this client. In that way it
+can distinguish between a resent connection request and an entirely
+new connection request.
'obd_connect_data'::
See <<struct-obd-connect-data>>.
-The target UUID is just "MGS", and the client UUID is set to the
-32byte string it gets from ... where?
-
-The 'struct lustre_handle' (the fourth buffer in the message) has its
-cookie set to .. what? It is set, but where does it come from?
+The 'ocd_connect_flags' field reflects the capabilities appropriate to
+the client. The 'ocd_brw_size' is set to the largest value for the
+size of an RPC that the client can handle. The 'ocd_ibits_known' and
+'ocd_checksum_types' values are set to what the client considers
+appropriate. Other fields in the descriptor and 'obd_connect_data'
+structures are zero.
-The 'ocd_connect_flags' field is set to (fixme: what?) reflecting the
-capabilities appropriate to the client. The 'ocd_brw_size' is set to the
-largest value for the size of an RPC that the client can handle. The
-'ocd_ibits_known' and 'ocd_checksum_types' values are set to what the client
-considers appropriate. Other fields in the descriptor and
-'obd_connect_data' structures are zero.
+.MGS_CONNECT Reply Packet Structure
+image::mgs-connect-reply.png["MGS_CONNECT Reply Packet Structure",height=50]
-Once the server receives the 'obd_connect_client' message on behalf of
-the given target it replies with an 'obd_connect_server' message. In
-that message the server sends the 'pb__handle' to uniquely
-identify the connection for subsequent communication. The client notes
-that handle in its import for the given target.
+//////////////////////////////////////////////////////////////////////
+The mgs-connect-reply.png diagram resembles this text art:
-fixme: Are there circumstances that could lead to the 'status'
-value in the reply being non-zero? What would lead to that and what
-error values would result?
+ MGS_CONNECT:
+ --reply---------------------------
+ | ptlrpc_body | obd_connect_data |
+ ----------------------------------
+//////////////////////////////////////////////////////////////////////
-The target maintains the last committed transaction for a client in
-its export for that client. If this is the first connection, then that
-last transaction value would just be zero. If there were previous
-transactions for the client, then the transaction number for the last
-such committed transaction is put in the 'pb_last_committed' field.
+'ptlrpc_body'::
+See <<struct-ptlrpc-body>> for details about the RPC descriptor. In a
+connect message the 'ptlrpc_body' field 'pb_handle', which is a
+<<struct-lustre-handle>>, is 0. That 'lustre_handle' is distinct from
+the one mentioned below. In a reply, the 'pb_handle' field has the
+value, $C_t$, uniquely identifying that target. The value $C_t$ is
+preserved in the 'imp_remote_handle' field of the client's import
+structure for this target. See <<struct-obd-import>>.
-In a connection request the operation is not file system modifying, so
-the 'pb_transno' value will be zero in the reply as well.
+'obd_connect_data'::
+See <<struct-obd-connect-data>>.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[[mgs-disconnect-rpc]]
-.MGS_DISCONNECT Generic Packet Structure
-image::mgs-disconnect-generic.png["MGS_DISCONNECT Generic Packet Structure",height=100]
+Disconnect from the MGS.
+
+.MGS_DISCONNECT Request Packet Structure
+image::mgs-disconnect-request.png["MGS_DISCONNECT Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The mgs-disconnect-generic.png diagram resembles this text art:
+The mgs-disconnect-request.png diagram resembles this text art:
MGS_DISCONNECT:
--request------
| ptlrpc_body |
---------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+.MGS_DISCONNECT Reply Packet Structure
+image::mgs-disconnect-reply.png["MGS_DISCONNECT Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mgs-disconnect-reply.png diagram resembles this text art:
+
+ MGS_DISCONNECT:
--reply--------
| ptlrpc_body |
---------------
//////////////////////////////////////////////////////////////////////
-The information exchanged in a DISCONNECT message is only that normally
-conveyed in the RPC descriptor, as described in <<struct-ptlrpc-body>>.
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
The mgs-connect-request.png diagram resembles this text art:
MGS_CONNECT:
- --request--------------------------------------------
- | ptlrpc_body | obd_uuid | obd_uuid | lustre_handle |
- -----------------------------------------------------
+ --request--------------------------------------------------
+ | ptlrpc_body | target_uuid | client_uuid | lustre_handle |
+ -----------------------------------------------------------
| obd_connect_data |
---------------------
//////////////////////////////////////////////////////////////////////
+See <<mgs-connect-rpc>>.
+
The client begins by carrying out the <<mgs-connect-rpc,MGS_CONNECT>>
-Lustre RPC, which establishes the connection (creates the
-import and the export) between the client and the MGS. The connect
-message from the client includes a 'lustre_handle' to uniquely
-identify itself. Subsequent request messages to the MGS will refer
-to that client-handle to verify that the client has already connected.
-The connection data from the client also proposes the set of
-<<connect-flags,connection flags>> appropriate to connecting to an MGS.
+Lustre RPC, which establishes the connection (creates the import and
+the export) between the client and the MGS. The connect message from
+the client includes a 'lustre_handle' to uniquely identify itself
+(note that this is distinct from the 'pb_handle' field of the
+structure described in <<struct-ptlrpc-body>>, which will be 0 in a
+connection request). The target notes the 'lustre_handle' value in its
+'export'. The connection data from the client also proposes the set of
+<<connect-flags,connection flags>> appropriate to connecting to an
+MGS.
.Flags for the client connection to an MGS
[options="header"]
----------------------------------
//////////////////////////////////////////////////////////////////////
-The MGS's reply to the connection request will include the handle that
-the server and client will both use to identify this connection in
-subsequent messages. This is the 'connection-handle' (as opposed to
-the client-handle mentioned a moment ago). The MGS also replies with
-the same set of connection flags.
+The MGS's reply to the connection request will include the
+'lustre_handle' (in the 'ptlrpc_body' structure's 'pb_handle' field)
+that the client will use (in its 'pb_handle') to identify this target
+in subsequent messages. The MGS also replies with the same set of
+connection flags.
Once the connection is established the client gets configuration
information for the file system from the MGS in four stages. First,
------------------------------
//////////////////////////////////////////////////////////////////////
+See <<ldlm-enqueue-rpc>>.
+
The client seeks a 'concurrent read' lock on the MGS.
*4 - The MGS sends an LDLM_ENQUEUE reply to the client.*
*5 - The Client sends an LLOG_ORIGIN_HANDLE_CREATE to the MGS.*
-
.LLOG_ORIGIN_HANDLE_CREATE Request Packet Structure
image::llog-origin-handle-create-request.png["LLOG_ORIGIN_HANDLE_CREATE Request Packet Structure",height=50]
-------------------------------------
//////////////////////////////////////////////////////////////////////
+See <<llog-origin-handle-create>>.
+
The first LLOG_ORIGIN_HANDLE_CREATE operation asks
for the security configuration file ("lfs-sptlrpc"). <<security>>
discusses security.
== -2, ENOENT) indicated that there was no such file, so nothing
actually gets read.
-Next, the client gets a configuration <<llog>> starting with a bitmap
-instructing it as to which among the configuration records on the MGS
-it needs. Another LDLM_ENQUEUE and LLOG_ORIGIN_HANDLE_CREATE pair of
-operations identifies the configuration client data ("lfs-client")
-file.
+Next, the client gets a configuration log (See <<llog>>) starting with
+a bitmap instructing it as to which among the configuration records on
+the MGS it needs. Another LDLM_ENQUEUE and LLOG_ORIGIN_HANDLE_CREATE
+pair of operations identifies the configuration client data
+("lfs-client") file.
*7 - The Client sends an LDLM_ENQUEUE to the MGS.*
+See <<ldlm-enqueue-rpc>>.
+
The client again seeks a 'concurrent read' lock on the MGS.
*8 - The MGS sends an LDLM_ENQUEUE reply to the client.*
*9 - The Client sends an LLOG_ORIGIN_HANDLE_CREATE to the MGS.*
+See <<llog-origin-handle-create>>.
+
The client asks for the 'lfs-client' log file, which holds a bitmap
indicating the available configuration records.
----------------------------
//////////////////////////////////////////////////////////////////////
+See <<llog-origin-handle-read-header>>.
+
The client asks for that log file's header. The MGS stores
configuration data in log records. A header at the beginning of
"lfs-client" uses a bitmap to identify the log records that are
----------------------------
//////////////////////////////////////////////////////////////////////
+See <<llog-origin-handle-next-block>>.
+
The client asks for the configuration record.
*14 - The MGS sends an LLOG_ORIGIN_HANDLE_NEXT_BLOCK reply to the client.*
*15 - The Client sends an LDLM_ENQUEUE to the MGS.*
+See <<ldlm-enqueue-rpc>>.
+
The client again seeks a 'concurrent read' lock on the MGS.
*16 - The MGS sends an LDLM_ENQUEUE reply to the client.*
*17 - The Client sends an MGS_CONFIG_READ to the MGS.*
+.MGS_CONFIG_READ Request Packet Structure
+image::mgs-config-read-request.png["MGS_CONFIG_READ Request Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mgs-config-read-request.png diagram resembles this text art:
+
+ MGS_CONFIG_READ:
+ --request------------------------
+ | ptlrpc_body | mgs_config_body |
+ ---------------------------------
+//////////////////////////////////////////////////////////////////////
+
+See <<mgs-config-read-rpc>>.
+
The client identifies the desired record in the 'lfs-cliir' file,
which contains the current details of the configuration for this file
system.
*18 - The MGS sends an MGS_CONFIG_READ reply to the client.*
+.MGS_CONFIG_READ Reply Packet Structure
+image::mgs-config-read-reply.png["MGS_CONFIG_READ Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mgs-config-read-reply.png diagram resembles this text art:
+
+ MGS_CONFIG_READ:
+ --reply-------------------------
+ | ptlrpc_body | mgs_config_res |
+ --------------------------------
+//////////////////////////////////////////////////////////////////////
+
The MGS responds with the actual configuration data. This gives the
client the list of all the servers and targets it will need to
communicate with.
*19 - The Client sends an LDLM_ENQUEUE to the MGS.*
+See <<ldlm-enqueue-rpc>>.
+
The client again seeks a 'concurrent read' lock on the MGS.
*20 - The MGS sends an LDLM_ENQUEUE reply to the client.*
*21 - The Client sends an LLOG_ORIGIN_HANDLE_CREATE to the MGS.*
+See <<llog-origin-handle-create>>.
+
The client asks for the 'params' log file.
*22 - The MGS sends an LLOG_ORIGIN_HANDLE_CREATE reply to the client.*
*23 - The Client sends an LLOG_ORIGIN_HANDLE_READ_HEADER to the MGS.*
+See <<llog-origin-handle-read-header>>.
+
The client asks for that log file's header.
*24 - The MGS sends an LLOG_ORIGIN_HANDLE_READ_HEADER reply to the client.*
The mds-connect-request.png diagram resembles this text art:
MDS_CONNECT:
- --request--------------------------------------------
- | ptlrpc_body | obd_uuid | obd_uuid | lustre_handle |
- -----------------------------------------------------
+ --request--------------------------------------------------
+ | ptlrpc_body | target_uuid | client_uuid | lustre_handle |
+ -----------------------------------------------------------
| obd_connect_data |
---------------------
//////////////////////////////////////////////////////////////////////
+See <<mds-connect-rpc>>.
+
As with the connect operation for the MGS, the connect message from
-the client includes a UUID to uniquely identify this connection, and
-subsequent messages to the lock manager on the server will refer to
-that UUID. The connection data from the client also proposes the set
+the client includes a 'lustre_handle' to uniquely identify this
+connection. The connection data from the client also proposes the set
of <<connect-flags,connection flags>> appropriate to connecting to an
MDS. The following are the flags always included.
The MDS replies to the connect message with a subset of the flags
proposed by the client, and the client notes those values in its
-import. The MDS's reply to the connection request will include a UUID
-that the server and client will both use to identify this connection
-in subsequent messages.
+import. As with the MGS, the MDS's reply to the connection request
+will include a 'lustre_handle' in the 'pb_handle' field that the
+client will use to identify this target in subsequent messages.
*3 - The Client sends an MDS_STATFS to the MDS.*
+.MDS_STATFS Request Packet Structure
+image::mds-statfs-request.png["MDS_STATFS Request Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-statfs-request.png diagram resembles this text art:
+
+ MDS_STATFS:
+ --request------
+ | ptlrpc_body |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
+See <<mds-statfs-rpc>>.
+
The client next uses an MDS_STATFS operation to request 'statfs'
information from the target, and that data is returned in the reply
message. The actual fields closely resemble the results of a 'statfs'
*4 - The MDS sends an MDS_STATFS reply to the client.*
-The MDS replies witht eh 'statfs' information.
+.MDS_STATFS Reply Packet Structure
+image::mds-statfs-reply.png["MDS_STATFS Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-statfs-reply.png diagram resembles this text art:
+
+ MDS_STATFS:
+ --reply---------------------
+ | ptlrpc_body | obd_statfs |
+ ----------------------------
+//////////////////////////////////////////////////////////////////////
+
+The MDS replies with the 'statfs' information.
*5 - The Client sends an MDS_GETSTATUS to the MDS.*
+.MDS_GETSTATUS Request Packet Structure
+image::mds-getstatus-request.png["MDS_GETSTATUS Request Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-getstatus-request.png diagram resembles this text art:
+
+ MDS_GETSTATUS:
+ --request-----------------
+ | ptlrpc_body | mdt_body |
+ --------------------------
+//////////////////////////////////////////////////////////////////////
+
+See <<mds-getstatus-rpc>>.
+
The client uses the MDS_GETSTATUS operation to request information
about the mount point of the file system.
*6 - The MDS sends an MDS_GETSTATUS reply to the client.*
+.MDS_GETSTATUS Reply Packet Structure
+image::mds-getstatus-reply.png["MDS_GETSTATUS Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-getstatus-reply.png diagram resembles this text art:
+
+ MDS_GETSTATUS:
+ --reply-------------------
+ | ptlrpc_body | mdt_body |
+ --------------------------
+//////////////////////////////////////////////////////////////////////
+
The server reply contains the 'fid' of the root directory of the file
system being mounted. If there is a security policy the capabilities
of that security policy are included in the reply.
*7 - The Client sends an MDS_GETATTR to the MDS.*
+.MDS_GETATTR Request Packet Structure
+image::mds-getattr-request.png["MDS_GETATTR Request Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-getattr-request.png diagram resembles this text art:
+
+ MDS_GETATTR:
+ --request-------------------------------
+ | ptlrpc_body | mdt_body | lustre_capa |
+ ----------------------------------------
+//////////////////////////////////////////////////////////////////////
+
+See <<mds-getattr-rpc>>.
+
The client then uses the MDS_GETATTR operation to get get further
information about the root directory of the file system. The request
message includes the above fid.
*8 - The MDS sends an MDS_GETATTR reply to the client.*
+.MDS_GETATTR Reply Packet Structure
+image::mds-getattr-reply.png["MDS_GETATTR Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-getattr-reply.png diagram resembles this text art:
+
+ MDS_GETATTR:
+ --reply------------------------------------------------
+ | ptlrpc_body | mdt_body | MDS_MD | ACL | lustre_capa |
+ -------------------------------------------------------
+ | lustre_capa |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
It will also include the security capability (if appropriate). The
reply also holds the same fid, and in this case the 'mdt_body' has
several additional fields filled in. These include the mtime, atime,
The ost-connect-request.png diagram resembles this text art:
OST_CONNECT:
- --request--------------------------------------------
- | ptlrpc_body | obd_uuid | obd_uuid | lustre_handle |
- -----------------------------------------------------
+ --request--------------------------------------------------
+ | ptlrpc_body | target_uuid | client_uuid | lustre_handle |
+ -----------------------------------------------------------
| obd_connect_data |
---------------------
//////////////////////////////////////////////////////////////////////
+See <<ost-connect-rpc>>.
+
As with the connect operations for the MGS and MDTs, the connect
-message from the client includes a UUID to uniquely identify this
-connection, and subsequent messages to the lock manager on the server
-will refer to that UUID. The connection data from the client also
+message from the client includes a 'lustre_handle' to uniquely
+identify this connection. The connection data from the client also
proposes the set of <<connect-flags,connection flags>> appropriate to
connecting to an OST. The following are the flags always included.
[[ost-connect-rpc]]
When a client initiates a connection to a specific target on an OSS,
-it does so via an OST_CONNECT RPC ('pb_oc' = 8). From a previous
-interaction with the MGS the client knows the UUID of the target OST,
-and can fill that value into the 'obd_connect_client' message.
+it does so via an OST_CONNECT RPC ('pb_opc' = 8).
-.OST_CONNECT Generic Packet Structure
-image::ost-connect-generic.png["OST_CONNECT Generic Packet Structure",height=100]
+.OST_CONNECT Request Packet Structure
+image::ost-connect-request.png["OST_CONNECT Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The ost-connect-generic.png diagram resembles this text art:
+The ost-connect-request.png diagram resembles this text art:
OST_CONNECT:
- --request--------------------------------------------
- | ptlrpc_body | obd_uuid | obd_uuid | lustre_handle |
- -----------------------------------------------------
- | obd_connect_data |
- ---------------------
- --reply---------------------------
- | ptlrpc_body | obd_connect_data |
- ----------------------------------
+ --request--------------------------------------------------
+ | ptlrpc_body | target_uuid | client_uuid | lustre_handle |
+ -----------------------------------------------------------
+ | obd_connect_data |
+ --------------------
//////////////////////////////////////////////////////////////////////
'ptlrpc_body'::
-RPC descriptor.
+RPC descriptor. See <<struct-ptlrpc-body>>. In a connect message the
+'ptlrpc_body' field 'pb_handle', which is a <<struct-lustre-handle>>,
+is 0. That 'lustre_handle' is distinct from the one mentioned
+below.
+
+'target_uuid'::
+A string identifying the target. See <<struct-obd-uuid>>. The client
+learns the UUID strings for the targets via an interaction with the
+MGS.
-'obd_uuid'::
-UUIDs of the target (first) and client (second) entities. See
-<<struct-obd-uuid>>.
+'client_uuid'::
+A string with the client's own UUID. This is also a
+<<struct-obd-uuid>>. The target sets the 'exp_client_uuid' field of
+its 'eport' structure to this value.
'lustre_handle'::
-See <<struct-lustre-handle>>.
+See <<struct-lustre-handle>>. This 'lustre_handle' is distinct from
+the 'pb_handle' field in the 'ptlrpc_body'. This 'lustre_handle'
+provied a unique 'cookie' to identify this client for this connection
+attempt. After a disconnect, a subsequent connect RPC will have a
+different value. The target preserves this cookie in the 'exp_handle'
+field of its 'obd_export' structure for this client. In that way it
+can distinguish between a resent connection request and an entirely
+new connection request.
'obd_connect_data'::
See <<struct-obd-connect-data>>.
-In the OST_CONNECT RPC reply from the server the 'ptlrpc_body' field
-'pb__handle' is set to uniquely identify the connection for subsequent
-communication. The client notes that handle in its import for the
-given target.
+.OST_CONNECT Reply Packet Structure
+image::ost-connect-reply.png["OST_CONNECT Reply Packet Structure",height=50]
+//////////////////////////////////////////////////////////////////////
+The ost-connect-reply.png diagram resembles this text art:
+ OST_CONNECT:
+ --reply---------------------------
+ | ptlrpc_body | obd_connect_data |
+ ----------------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>. The 'pb_handle' field has
+the value, $C_t$, uniquely identifying that target. The value $C_t$ is
+preserved in the 'imp_remote_handle' field of the client's import
+structure for this target. See <<struct-obd-import>>.
+
+'obd_connect_data'::
+See <<struct-obd-connect-data>>.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[[ost-disconnect-rpc]]
-.OST_DISCONNECT Generic Packet Structure
-image::ost-disconnect-generic.png["OST_DISCONNECT Generic Packet Structure",height=100]
+Disconnecting from an OST.
+
+.OST_DISCONNECT Request Packet Structure
+image::ost-disconnect-request.png["OST_DISCONNECT Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The ost-disconnect-generic.png diagram resembles this text art:
+The ost-disconnect-request.png diagram resembles this text art:
OST_DISCONNECT:
--request------
| ptlrpc_body |
---------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+.OST_DISCONNECT Reply Packet Structure
+image::ost-disconnect-reply.png["OST_DISCONNECT Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ost-disconnect-reply.png diagram resembles this text art:
+
+ OST_DISCONNECT:
--reply--------
| ptlrpc_body |
---------------
//////////////////////////////////////////////////////////////////////
-The information exchanged in an OST_DISCONNECT request message is
-only that normally conveyed in the RPC descriptor, as described in
-<<struct-ptlrpc-body>>.
-
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
An RPC that modifies the size attribute of a resource on an OST.
-.OST_PUNCH Generic Packet Structure
-image::ost-punch-generic.png["OST_PUNCH Generic Packet Structure",height=100]
+.OST_PUNCH Request Packet Structure
+image::ost-punch-request.png["OST_PUNCH Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The ost-punch-generic.png diagram resembles this text art:
+The ost-punch-request.png diagram resembles this text art:
OST_PUNCH:
--request-------------------------------
| ptlrpc_body | ost_body | lustre_capa |
----------------------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'ost_body'::
+Metadata about the resource. See <struct-ost-body>>.
+
+include::struct_lustre_capa.txt[]
+
+.OST_PUNCH Reply Packet Structure
+image::ost-punch-reply.png["OST_PUNCH Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ost-punch-reply.png diagram resembles this text art:
+
+ OST_PUNCH:
--reply-------------------
| ptlrpc_body | ost_body |
--------------------------
//////////////////////////////////////////////////////////////////////
'ptlrpc_body'::
-RPC descriptor.
+RPC descriptor. See <<struct-ptlrpc-body>>.
'ost_body'::
Metadata about the resource, as it appears on the OST.
-'lustre_capa'::
-So called "capabilities" structure. This is deprecated in recent
-versions of Lustre, and commonly appears in the packet header as a zero
-length buffer.
-
OST_SETATTR (pb_opc = 2) is an RPC that sets resource attributes.
-.OST_SETATTR Generic Packet Structure
-image::ost-setattr-generic.png["OST_SETATTR Generic Packet Structure",height=100]
+.OST_SETATTR Request Packet Structure
+image::ost-setattr-request.png["OST_SETATTR Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The ost-setattr-generic.png diagram resembles this text art:
+The ost-setattr-request.png diagram resembles this text art:
OST_SETATTR:
--request-----------------
| ptlrpc_body | ost_body |
--------------------------
- --reply-------------------
- | ptlrpc_body | ost_body |
- --------------------------
//////////////////////////////////////////////////////////////////////
include::struct_ptlrpc_body.txt[]
include::struct_ost_body.txt[]
+.OST_SETATTR Reply Packet Structure
+image::ost-setattr-reply.png["OST_SETATTR Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ost-setattr-reply.png diagram resembles this text art:
+
+ OST_SETATTR:
+ --reply-------------------
+ | ptlrpc_body | ost_body |
+ --------------------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+'ost_body'::
+Metadata about the resource. See <struct-ost-body>>.
The reply message conveys 'statfs' data when it succeeds, and an error
code if it doesn't.
-.OST_STATFS Generic Packet Structure
-image::ost-statfs-generic.png["OST_STATFS Generic Packet Structure",height=100]
+.OST_STATFS Request Packet Structure
+image::ost-statfs-request.png["OST_STATFS Request Packet Structure",height=50]
//////////////////////////////////////////////////////////////////////
-The ost-statfs-generic.png diagram resembles this text art:
+The ost-statfs-request.png diagram resembles this text art:
OST_STATFS:
--request------
| ptlrpc_body |
---------------
+//////////////////////////////////////////////////////////////////////
+
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
+
+.OST_STATFS Reply Packet Structure
+image::ost-statfs-reply.png["OST_STATFS Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ost-statfs-reply.png diagram resembles this text art:
+
+ OST_STATFS:
--reply---------------------
| ptlrpc_body | obd_statfs |
----------------------------
//////////////////////////////////////////////////////////////////////
-'ptlrpc_body':: RPC descriptor. Only the 'pb_opc' value (OST_STATFS =
-41) is directly relevant to the OST_STATFS request message. The rest
-of the 'ptlrpc_body' fields handle generic information about the
-RPC, as discussed in <<struct-ptlrpc-body>>, including generic error
-conditions. In a normal reply ('pb_type' = PTL_RPC_MSG_REPLY) the
-'pb_status' field is 0. The one error that can be returned in
-'pb_status' that is speficially from OST_STATFS' handling is -ENOMEM,
-which occurs if there is not enough memory to allocate a temporary
-buffer for the 'statfs' data.
+'ptlrpc_body'::
+RPC descriptor. See <<struct-ptlrpc-body>>.
include::struct_obd_statfs.txt[]
--- /dev/null
+RPC
+~~~
+
+The most common activity in the Lustre protocol is for a client to
+initiate a Remote Procedure Call (RPC) directed at a specific
+target. There are two additional activities supported by the Lustre
+protocol. A server may initiate an RPC to the target on another
+server, such as an MDS RPC to the MGS for configuration data, or an
+RPC to an OST to update the MDS's state with available space
+data. Finally, in a "call-back" RPC the service side (where the target
+resides) initiates the RPC, which may be directed at a client or
+another server.
~~~~~~~~
[[security]]
-The discussion of security is deferred until later. It is a side issue
-to most of what is being documented here. It is seldom employed, and
-may have older and buggy code.
+The discussion of security is deferred until later.
------------------------------------------------
//////////////////////////////////////////////////////////////////////
+See <<mds-reint-rpc>>.
+
In this case the 'setattr' wants to set the mode attribute on
the resource. The 'mdt_rec_setattr' identifies the resource with the
'sa_fid' field, and the 'sa_valid' field is set to 0x2041:
The RPC(s) that get sent for the 'setattr' depend on specifically what
values are being set. If the time values are being set (as in a
"touch" command) then there are RPCs in addition to the MDS_REINT,
-with the REINT_SETATTR sub-operation, that update the time vales on
+with the REINT_SETATTR sub-operation, that update the time values on
the MDT. That operation is followed by an OST_SETATTR that sets the
time values on the OST (or OSTs if there are several). But in order to
know what OSTs to contact the client must first get the layout of the
*1 - The client issues an MDS_REINT with the REINT_SETATTR
sub-operation.*
+See <<mds-reint-rpc>>.
+
The MDS_REINT request RPC closely resembles the one described above,
but in this case the 'setattr' wants to set the time attributes on the
resource. The 'mdt_rec_setattr' again identifies the resource with the
-----------------------------------------------------------
//////////////////////////////////////////////////////////////////////
+See <<ldlm-enqueue-rpc>>.
+
The 'ldlm_request' asks for a read lock on the resource and has its
intent flag set. The 'ldlm_intent' has the intent opcode is 0x800:
IT_LAYOUT. The 'layout_intent' has the 'li_opc' value 0:
--------------------------
//////////////////////////////////////////////////////////////////////
-The 'ost_body' structure is documented in <<struct-ost-body>>. In
-this case the 'o_valid' field is 0x300400f, so the valid fields are
-given by:
+See <<ost-setattr-rpc>>.
+
+The 'ost_body' structure is documented in the <<struct-ost-body>>
+section. In this case the 'o_valid' field is 0x300400f, so the valid
+fields are given by:
.Flags for 'o_valid' field of 'struct os_body'
[options="header"]
7 <--------LDLM_CANCEL
8 <-----------LDLM_CP_CALLBACK
9 LDLM_CP_CALLBACK----------->
-10 OST_PUNCH-------------------->
+10 OST_PUNCH-------------------->
11 LDLM_CANCEL------>
12 <--------------------OST_PUNCH
//////////////////////////////////////////////////////////////////////
*1 - The client issues an MDS_REINT with the REINT_SETATTR
sub-operation.*
+See <<mds-reint-rpc>>.
+
The MDS_REINT request RPC closely resembles the one described above,
but in this case the 'setattr' wants to modify the size attribute on the
resource. The 'mdt_rec_setattr' again identifies the resource with the
*3 - The client asks the OST for a write lock of type LDLM_EXTENT.*
+See <<ldlm-enqueue-rpc>>.
+
The 'ldlm_request' asks for a write lock with the lock descriptor
resource's type set to LDLM_EXTENT, the policy data covering the whole
file, and the lock handle set to identify this request. The rest of
------------------------------
//////////////////////////////////////////////////////////////////////
+See <<ldlm-bl-callback-rpc>>.
+
*5 - Client2 acknowledges the request and returns the lock.*
The LDLM_BL_CALLBACK is an "empty" RPC in that it only has the
------------------------------
//////////////////////////////////////////////////////////////////////
+See <<ldlm-cancel-rpc>>.
+
The 'ldlm_request' indicates which lock is being canceled in its
(first) 'lock_handle' field. The OST then looks for anyone else
waiting on that lock, which it finds is Client1. It waits to reply to
----------------------------------------
//////////////////////////////////////////////////////////////////////
+See <<ldlm-cp-callback-rpc>>.
+
*9 - Client1 acknowledges the lock update.*
The reply is "empty" in this case as well. The opcode in the
--------------------------
//////////////////////////////////////////////////////////////////////
+See <<ost-punch-rpc>>.
+
In this case the 'o_valid' field is 0x30403d:
.Flags for 'o_valid' field of 'struct os_body'
-------------------------------------------------
//////////////////////////////////////////////////////////////////////
+See <<mds-reint-rpc>>.
+
The 'setxattr' wants to set the 'ctime' associated with the EA update
as well, so the one 'sx_valid' flag it uses is OBD_MD_FLCTIME
(0x00008). The 'sx_valid' field also has the OBD_MD_FLXATTR
~~~~~~
The 'statfs' VFS method queries file-system-wide space and inode
-usage. For details about the MDS_STATFS RPC, including possible faults
-and error return codes, see <<mds-statfs-rpc>>, and for OST_STATFS see
-<<ost-statfs-rpc>>.
+usage. For details about the MDS_STATFS RPC see <<mds-statfs-rpc>>,
+and for OST_STATFS see <<ost-statfs-rpc>>.
A client gets 'statfs' information for the file system as a whole by
-first querying the individual storage targets for the 'statfs'
-information on each back-end file system. The RPCs that flow are as
-shown in <<statfs-rpcs>>. The values returned in the VFS call are
-built from the results of the *_STATFS RPCs as follows:
+first querying the individual storage targets (OSDs) for their
+'statfs' information. The RPCs that flow are as shown in
+<<statfs-rpcs>>. The values returned in the VFS call are built from
+the results of the *_STATFS RPCs as follows:
.Statfs values
[options="header"]
The MDS_STATFS request is a so-called 'empty' RPC in that it consists
only of the 'ptlrpc_body' (Lustre RPC descriptor) with the opcode set to
-MDS_STATFS (41). Refer to the discussion of <<mds-statfs-rpc>> for
-more general information about the MDS_STATFS RPC's request and reply
-messages.
+MDS_STATFS (41).
.MDS_STATFS Request Packet Structure
+image::mds-statfs-request.png["MDS_STATFS Request Packet Structure",height=50]
-:frame: none
-:grid: none
-[width="50%", cols="2a"]
-|====
-| request
-[cols="1"]
-!===================
-! <<struct-ptlrpc-body,ptlrpc_body>> !
-!===================
-|====
+//////////////////////////////////////////////////////////////////////
+The mds-statfs-request.png diagram resembles this text art:
+
+ MDS_STATFS:
+ --request------
+ | ptlrpc_body |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
+See <<mds-statfs-rpc>>.
*2 - The MDS_STATFS reply returns 'statfs' info*
<<struct-obd-statfs>>.
.MDS_STATFS Reply Packet Structure
+image::mds-statfs-reply.png["MDS_STATFS Reply Packet Structure",height=50]
-:frame: none
-:grid: none
-[width="50%", cols="2a"]
-|====
-| reply
-[cols="2"]
-!===================
-! <<struct-ptlrpc-body,ptlrpc_body>> ! <<struct-obd-statfs,obd_statfs>> !
-!===================
-|====
+//////////////////////////////////////////////////////////////////////
+The mds-statfs-reply.png diagram resembles this text art:
+
+ MDS_STATFS:
+ --reply---------------------
+ | ptlrpc_body | obd_statfs |
+ ----------------------------
+//////////////////////////////////////////////////////////////////////
*3 - The client issues an OST_STATFS request to each OST.*
MDT.
.OST_STATFS Request Packet Structure
+image::ost-statfs-request.png["OST_STATFS Request Packet Structure",height=50]
-:frame: none
-:grid: none
-[width="50%", cols="2a"]
-|====
-| request
-[cols="1"]
-!===================
-! <<struct-ptlrpc-body,ptlrpc_body>> !
-!===================
-|====
+//////////////////////////////////////////////////////////////////////
+The ost-statfs-request.png diagram resembles this text art:
+
+ OST_STATFS:
+ --request------
+ | ptlrpc_body |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+See <<mds-statfs-rpc>>.
*4 - The OST_STATFS reply returns 'statfs' info*
<<struct-obd-statfs>>.
.OST_STATFS Reply Packet Structure
+image::ost-statfs-reply.png["OST_STATFS Reply Packet Structure",height=50]
-:frame: none
-:grid: none
-[width="50%", cols="2a"]
-|====
-| reply
-[cols="2"]
-!===================
-! <<struct-ptlrpc-body,ptlrpc_body>> ! <<struct-obd-statfs,obd_statfs>> !
-!===================
-|====
+//////////////////////////////////////////////////////////////////////
+The ost-statfs-reply.png diagram resembles this text art:
+
+ OST_STATFS:
+ --reply---------------------
+ | ptlrpc_body | obd_statfs |
+ ----------------------------
+//////////////////////////////////////////////////////////////////////
-Layout Intent
-^^^^^^^^^^^^^
+.Layout Intent
[[struct-layout-intent]]
-
+****
An LDLM_ENQUEUE RPC with a layout intent uses the 'layout_intent'
structure to specify the desired use for the layout.
The other fields - 'li_flags', 'li_start', and 'li_end' - are reserved for
future use.
+****
-LDLM Intent
-^^^^^^^^^^^
+.LDLM Intent
[[struct-ldlm-intent]]
-
+****
A lock request can include an 'intent' operation. Which operation is
encoded in the 'ldlm_intent' 'opc'.
#define IT_QUOTA_CONN (1 << 12)
#define IT_SETXATTR (1 << 13)
----
-
+****
-LDLM Reply
-^^^^^^^^^^
+.LDLM Reply
[[struct-ldlm-reply]]
-
+****
The 'ldlm_reply' structure is the reciprocal of the 'ldlm_request'.
[source,c]
it is reporting back to the initiator of the request (in the
'lock_desc') both the lock that was actually granted, if any, and lock
server's (possibly) updated value for what was being requested.
-
+****
The 'lock_count' field represents how many requests are queued on this
resource.
+.LDLM Lock Descriptor
[[struct-ldlm-lock-desc]]
-
+****
The lock descriptor conveys the specific details about a particular
lock being requested or granted. It appears in
<<struct-ldlm-request>>.
ldlm_wire_policy_data_t l_policy_data;
};
----
+****
+.LDLM Resource Descriptor
[[struct-ldlm-resource-desc]]
-
+****
The resource descriptor identifies the individual resource that is
being locked, along with what sort of thing it is.
struct ldlm_res_id lr_name;
};
----
+****
-[[ldlm-type-t]]
The 'lr_type' field identifies one of the four types of lock that
might be placed on a resource. A "plain" lock type just locks a
particular resource. An "extent" lock type only locks a contiguous
descriptor may also have no type at all, in which case the 'lr_type'
field is 0, meaning "no lock".
+.LDLM Type
+[[ldlm-type-t]]
+****
[source,c]
----
enum {
LDLM_IBITS = 13,
} ldlm_type_t;
----
+****
-[[struct-ldlm-res-id]]
-The 'lr_name' field identifies (by name) the resource(s) that are the
-objects of the locking operation.
+The 'lr_name' field of the 'ldlm_resource_desc' (See
+<<struct-ldlm-res-id>>) field identifies the resource that is the
+object of the locking operation.
+.LDLM Resource ID
+[[struct-ldlm-res-id]]
+****
[source,c]
----
struct ldlm_res_id {
};
----
-The 'name' array holds identifiers for the resource in question. Those
-identifiers may be the elements of a 'struct lu_fid' file ID, or they
-may be other uniquely identifying values for the resource. See
-<<struct-lu-fid>>.
+The 'name' identifies for the resource in question. It may be a FID
+(See <<struct-lu-fid>>), or they may be other uniquely identifying
+values for the resource.
+****
-[[ldlm-mode-t]]
The 'l_req_mode' and 'l_granted_mode' fields give the kind of lock
being requested and the kind of lock that has been granted. The field
values are:
+.LDLM Mode
+[[ldlm-mode-t]]
+****
[source,c]
----
enum {
Despite the fact that the lock modes are not overlapping, these lock
modes are exclusive. In addition the mode value 0 is the MINMODE,
i.e. no lock at all.
+****
-In a request 'l_req_mode' is the value actually being requested and
-'l_granted_mode' is the value that currently is in place on for the
-requester. In a reply the 'l_req_mode' may be modified if more or
-fewer privileges were granted than requested, and the
-'l_granted_mode' is what has, in fact, been granted.
+In an 'ldlm_request' structure the 'l_req_mode' field is the value
+actually being requested and the 'l_granted_mode' field is the value
+that currently is in place on for the requester. In an 'ldlm_reply'
+the 'l_req_mode' field may be modified if more or fewer privileges
+were granted than requested, and the 'l_granted_mode' is what has, in
+fact, been granted.
-[[ldlm-wire-policy-data-t]]
The 'l_policy_data' field gives the kind of resource being
requested/granted. It is a union of these struct definitions:
+.LDLM Wire Policy Data
+[[ldlm-wire-policy-data-t]]
+****
[source,c]
----
typedef union {
struct ldlm_inodebits l_inodebits;
} ldlm_wire_policy_data_t;
----
+****
+.LDLM Extent
[[struct-ldlm-extent]]
+****
[source,c]
----
struct ldlm_extent {
__u64 gid;
};
----
+****
+.LDLM Flock Wire
[[struct-ldlm-flock-wire]]
+****
[source,c]
----
struct ldlm_flock_wire {
__u32 lfw_pid;
};
----
+****
+.LDLM Inodebits
[[struct-ldlm-inodebits]]
+****
[source,c]
----
struct ldlm_inodebits {
__u64 bits;
};
----
+****
Thus the lock may be on an 'extent', a contiguous sequence of bytes
in a regular file; an 'flock wire', whatever to heck that is; or a
The lgd_llh_flags are:
+
+.LLog Flags
+****
[source,c]
----
enum llog_flag {
LLOG_F_IS_PLAIN = 0x4,
};
----
+****
-LLog Record Header
-^^^^^^^^^^^^^^^^^^
+.LLog Record Header
[[struct-llog-rec-hdr]]
+****
[source,c]
----
struct llog_rec_hdr {
__u32 lrh_id;
};
----
+****
The 'llog_rec_hdr' is at the start of each llog record and describes
the log record. 'lrh_len' holds the record size in bytes, including
for an arbitrary record within the file. 'lrh_type' describes the type
of data stored in this record.
+.LLog Operation Type
+****
[source,c]
----
enum llog_op_type {
LLOG_LOGID_MAGIC = LLOG_OP_MAGIC | 0x4553b,
};
----
+****
-LLog Record Tail
-^^^^^^^^^^^^^^^^
+.LLog Record Tail
[[struct-llog-rec-tail]]
+****
[source,c]
----
struct llog_rec_tail {
__u32 lrt_index;
};
----
+****
The 'llog_rec_tail' is at the end of each llog record. The 'lrt_len'
and 'lrt_index' fields must be the same as 'lrh_len' and 'lrh_index'
-LLOGD Body
-^^^^^^^^^^
+.LLOGD Body
[[struct-llogd-body]]
-
+****
[source,c]
----
struct llogd_body {
__u64 lgd_cur_offset;
};
----
+****
+.LLog LogID
[[struct-llog-logid]]
-
+****
[source,c]
----
struct llog_logid {
__u32 lgl_ogen;
};
----
+****
The 'llog_logid' structure is used to identify a single Lustre log file.
It holds a <<struct-ost-id>> in 'lgl_oi', which is typically a FID.
entries in the 'lmm_objects' array, which can be determined by the overall
layout size.
+.LOV OST Data
+[[struct-lov-ost-data-v1]]
+****
[source,c]
----
struct lov_ost_data_v1 { /* per-stripe data structure (little-endian)*/
__u32 l_ost_idx; /* OST index in LOV (lov_tgt_desc->tgts) */
};
----
-[[struct-lov-ost-data-v1]]
+****
The 'lov_ost_data_v1' structure is used for both LOV_MAGIC_V1 and
LOV_MAGIC_V3 types to describe the OST objects allocated to this
-Lustre File Identifiers
-^^^^^^^^^^^^^^^^^^^^^^^
-[[struct-lu-fid]]
-
-Each resource stored on a target is assigned an identifier that is
-unique to that resource.
+.Lustre FID
+[[struct-lu-fid]]
+****
[source,c]
--------
struct lu_fid {
};
--------
-A file identifier ('FID') is a 128-bit numbers that uniquely identifies a
-single file or directory on an MDTs or OSTs within a single Lustre file
-system. The FID for a Lustre file or directory is the FID from the
-corresponding MDT entry for the file. Each of the data resource for that
-file will also have a FID for each corresponding OST resource on which the
-file stores data.
+A file identifier ('FID') is a 128-bit number that uniquely identifies a
+resource on an MDT or OST. The FID for the resource on an MDT
+identifies the file as a whole. The FID for a resource on an OST
+identifiess the fragment of a file assigned to that OST.
The 'f_seq' field holds the sequence number, or SEQ, and is used in
conjunction with the 'FID location database' (FLDB) to
sequence to identify a particular object.
The 'f_ver' value identifies which version of a resource is being
-identified, in the event that the resource is being updated, and
+identified in the event that the resource is being updated, and
different hosts might be referring to different versions of the same
resource. It has never been used as of Lustre 2.8.
+****
+.FID SEQ
[source,c]
----
enum fid_seq {
FID_SEQ_IGIF_MAX = 0x0ffffffffULL,
FID_SEQ_IDIF = 0x100000000ULL,
FID_SEQ_IDIF_MAX = 0x1ffffffffULL,
- /* Normal FID sequence starts from this value, i.e. 1<<33 */
FID_SEQ_START = 0x200000000ULL,
- /* sequence for local pre-defined FIDs listed in local_oid */
FID_SEQ_LOCAL_FILE = 0x200000001ULL,
FID_SEQ_DOT_LUSTRE = 0x200000002ULL,
- /* sequence is used for local named objects FIDs generated
- * by local_object_storage library */
FID_SEQ_LOCAL_NAME = 0x200000003ULL,
- /* Because current FLD will only cache the fid sequence, instead
- * of oid on the client side, if the FID needs to be exposed to
- * clients sides, it needs to make sure all of fids under one
- * sequence will be located in one MDT. */
FID_SEQ_SPECIAL = 0x200000004ULL,
FID_SEQ_QUOTA = 0x200000005ULL,
FID_SEQ_QUOTA_GLB = 0x200000006ULL,
FID_SEQ_ROOT = 0x200000007ULL, /* Located on MDT0 */
FID_SEQ_LAYOUT_RBTREE = 0x200000008ULL,
- /* sequence is used for update logs of cross-MDT operation */
FID_SEQ_UPDATE_LOG = 0x200000009ULL,
- /* Sequence is used for the directory under which update logs
- * are created. */
FID_SEQ_UPDATE_LOG_DIR = 0x20000000aULL,
FID_SEQ_NORMAL = 0x200000400ULL,
FID_SEQ_LOV_DEFAULT = 0xffffffffffffffffULL
};
----
+****
There are several reserved ranges of FID sequence values, to allow for
interoperability with older Lustre filesystems, to identify "well known"
objects for internal or external use, as well as for future expansion.
The 'FID_SEQ_NORMAL' (0x200000400+) range is used for normal object
identifiers. These objects are visible in the namespace if allocated
by an MDT, or may be OST objects.
+****
--- /dev/null
+.Lustre Capability
+[[struct-lustre-capa]]
+****
+The so-called "capabilities" structure is deprecated in recent
+versions of Lustre, and commonly appears in the packet header as a zero
+length buffer. It conveys security capabilities. See <<security>>.
+----
+struct lustre_capa {
+ struct lu_fid lc_fid; /** fid */
+ __u64 lc_opc; /** operations allowed */
+ __u64 lc_uid; /** file owner */
+ __u64 lc_gid; /** file group */
+ __u32 lc_flags; /** HMAC algorithm & flags */
+ __u32 lc_keyid; /** key# used for the capability */
+ __u32 lc_timeout; /** capa timeout value (sec) */
+ __u32 lc_expiry; /** expiry time (sec) */
+ __u8 lc_hmac[CAPA_HMAC_MAX_LEN]; /** HMAC */
+};
+----
+****
-Lustre Handle
-^^^^^^^^^^^^^
+.Lustre Handle
[[struct-lustre-handle]]
-
-A 'lustre_handle' is an opaque cookie that identifies some local object
-to another node or another layer in the software stack. It is not the
-physical address of the data structure in memory to avoid memory
-corruption in case the object has been freed, but rather a cookie in a
-lookup table that provides a layer of indirection that can safely
-determine if the object still exists or not.
-
-A 'lustre_handle' can identify a variety of different types of objects,
-such as client-target connections, open file handles, DLM lock handles,
-etc. The meaning of the handle is dependent on the context in which it
-is used.
+****
+A 'lustre_handle' is an opaque cookie that identifies some local
+object to another node or another layer in the software stack. It can
+identify a variety of different types of objects, such as
+client-target connections, open file handles, DLM lock handles, etc.
+The meaning of the handle is dependent on the context in which it is
+used.
[source,c]
----
__u64 cookie;
};
----
-
+****
The 'mbo_max_mdsize' field indicates the maximu size of the file layout,
as described in <<struct-lov-mds-md>>
-The 'mbo_max_cookiesize' field is unused since Lustre 2.4. It formerly
-held the maximum permissible size of llog cookies for destroyed OST objectb
-
+The 'mbo_max_cookiesize' field is unused since Lustre 2.4. It
+formerly held the maximum permissible size of llog cookies for
+destroyed OST objects.
The 'rr_opcode' field defines one among the several sub-commands for
MDS REINT RPCs. Those opcodes are:
+.MDS Reint Type
+****
[source,c]
----
typedef enum {
REINT_SETXATTR = 7,
} mds_reint_t;
----
+****
Based on that opcode one of the variants of the structure will
actually be used. See <<mds-reint-setattr-rpc>> for one example.
The 'rr_suppgid1' and 'rr_suppgid2' fields are supplementary GID
information for kernels that have it.
-The 'rr_fid1' and 'rr_fid2' fields specify the file IDs for the
-resources being operated upon. If only one resource is being acted
-upon then 'rr_fid2' is not used.
+The 'rr_fid1' and 'rr_fid2' fields specify the FIDs (See
+<<struct-lu-fid>>) for the resources being operated upon. If only one
+resource is being acted upon then 'rr_fid2' is not used.
The 'rr_mtime' 'rr_atime' and 'rr_ctime' fields give the object
timestamps in seconds for the last modification time, the last
The 'rr_bias' field adds additional optional information to the
REINT. The possible values are:
+.MDS Operation Bias
+****
[source,c]
----
enum mds_op_bias {
MDS_HSM_RELEASE = 1 << 12,
};
----
+****
For example, MDS_DATA_MODIFIED signals to the HSM system that the MDT
should set the corresponding HSM extended attribute. We'll return to
There is only one FID to be operated upon, and the 'sa_valid',
'sa_uid', and 'sa_gid' values take the place of the second 'struct
-lu_fid'.
+lu_fid' in the generic <<struct-mdt-rec-reint>>. See also
+<<struct-lu-fid>>.
The 'sa_valid' field identifies which of the other fields in the
structure are to be honored. If the corresponding flag bit is not set
then the value of the corresponding field is to be ignored. The flags
are:
+.SA Valid
+****
[source,c]
----
#define MDS_ATTR_MODE 0x1ULL /* = 1 */
#define MDS_ATTR_FROM_OPEN 0x4000ULL /* = 16384 */
#define MDS_ATTR_BLOCKS 0x8000ULL /* = 32768 */
----
+****
The 'sa_uid' and 'sa_gid' fields give the values of the UID and GID
attributes, respectively, to be set on the target resource.
The 'setxattr' variant modifies the semantics of the generic REINT
fields as follows:
-There is only one FID to be operated upon, and the 'sx_valid',
-and the second 'struct lu_fid' is just "padding".
+There is only one FID to be operated upon, and the 'sx_valid', and the
+second 'struct lu_fid' from the generic <<struct-mdt-rec-reint>> is
+just "padding". See also <<struct-lu-fid>>.
The 'sx_valid' field identifies which of the other fields in the
structure are to be honored. If the corresponding flag bit is not set
values draw from the same set of definitions as
<<struct-mdt-rec-setattr>>.
+.SX Valid
+****
.Flags for 'sx_valid' field of 'struct mdt_rec_setxattr'
[options="header"]
|====
| Flag | Meaning
| OBD_MD_FLCTIME | ctime attribute
|====
+****
The 'sx_time' field is set to the 'ctime' value for the update and the
OBD_MD_FLCTIME value is used in the 'sx_valid' field to indicate the
-MGS Configuration Data
-^^^^^^^^^^^^^^^^^^^^^^
-
+.MGS Configuration Data
+[[struct-mgs-config-body]]
+****
[source,c]
----
#define MTI_NAME_MAXLEN 64
'mcb_bits' is the log2 of the units of minimum bulk transfer size,
typically 4096 or 8192 bytes, while 'mcb_units' is the maximum number of
2^mcb_bits sized units that can be transferred in a single request.
-
-[source,c]
-----
-struct mgs_config_res {
- __u64 mcr_offset; /* index of last config log */
- __u64 mcr_size; /* size of the log */
-};
-----
-
-The 'mgs_config_res' structure returns information describing the
-replied configuration llog data requested in 'mgs_config_body'.
-'mcr_offset' contains the last configuration record number returned
-by this reply. 'mcr_size' contains the maximum record index in the
-entire configuration llog. When 'mcr_offset' equals 'mcr_size' there
-are no more records to process in the log.
-
+****
--- /dev/null
+.MGS Configuration Result
+[[struct-mgs-config-res]]
+****
+[source,c]
+----
+struct mgs_config_res {
+ __u64 mcr_offset; /* index of last config log */
+ __u64 mcr_size; /* size of the log */
+};
+----
+
+The 'mgs_config_res' structure returns information describing the
+replied configuration llog data requested in 'mgs_config_body'.
+'mcr_offset' contains the last configuration record number returned
+by this reply. 'mcr_size' contains the maximum record index in the
+entire configuration llog. When 'mcr_offset' equals 'mcr_size' there
+are no more records to process in the log.
+****
OBD Connect Data
^^^^^^^^^^^^^^^^
-[[struct-obd-connect-data]]
-
An 'obd_connect_data' structure accompanies every connect operation in
both the request message and in the reply message. It contains the
mutually supported features that are negotiated between the client and
server at *_CONNECT time.
-At *_CONNECT time the client sends in its request 'ocd_connect_flags'
-the flags for all features that it understands and intends to use (for
-example 'OBD_CONNECT_RDONLY' is optional depending on client mount
-options). The request also contains other fields that are only valid
-if the matching flag is set. The server replies in 'ocd_connect_flags'
-with the subset of feature flags that it understands and intends to honour.
-The server may set fields in the reply for mutually-understood features.
-
+.The OBD Connect Data Structure
+[[struct-obd-connect-data]]
+****
[source,c]
----
struct obd_connect_data {
__u64 paddingF;
};
----
+****
+
+At *_CONNECT time the client sends in its request 'ocd_connect_flags'
+the flags for all features that it understands and intends to use (for
+example 'OBD_CONNECT_RDONLY' is optional depending on client mount
+options). The request also contains other fields that are only valid
+if the matching flag is set. The server replies in 'ocd_connect_flags'
+with the subset of feature flags that it understands and intends to honour.
+The server may set fields in the reply for mutually-understood features.
The 'ocd_connect_flags' field encodes the connect flags giving the
capabilities of a connection between client and target. Several of
server then replies with the subset of those capabilities it agrees to
honor (for the given target).
-If the 'OBD_CONNECT_VERSION' flag is set then the 'ocd_version' field is
-valid. The 'ocd_version' gives an encoding of the Lustre
-version. For example, Version 2.7.55 would be hexadecimal number
-0x02075500.
+If the 'OBD_CONNECT_VERSION' flag is set then the 'ocd_version' field
+is valid. The 'ocd_version' gives an encoding of the Lustre
+version. The Lustre version itself is a four-tuple of decimal numbers
+(major, minor, patch, fix), with missing trailing values defaulting to
+zero. Each of the four numbers is shifted into place in the four bytes
+of the 'ocd_version'. For example, Version 2.7.55 would be
+hexadecimal number 0x02073700.
If the OBD_CONNECT_GRANT flag is set then the 'ocd_grant' field is
valid. The 'ocd_grant' value in a reply (to a connection request)
-sets the client's grant.
+initializes the client's grant. See <<grant>>.
If the OBD_CONNECT_INDEX flag is set then the 'ocd_index' field is
valid. The 'ocd_index' value is set in a request to hold the LOV
-index of the OST or the LMV index of the MDT. The server should
-refuse connections to targets for which the 'ocd_index' does not
-match the actual target index.
+index of the OST or the LMV index of the MDT. The server's export for
+the target holds the correct value, and if the client send a value
+that does not match the server returns the -EBDF error.
If the OBD_CONNECT_BRW_SIZE flag is set then the 'ocd_brw_size' field
is valid. The 'ocd_brw_size' value sets the maximum supported bulk
If the OBD_CONNECT_IBITS flag is set then the 'ocd_ibits_known' field
is valid. The 'ocd_ibits_known' flags determine the handling of
locks on MDS inodes. The OBD_CONNECT_IBITS flag was introduced in
-Lustre 1.4 and is mandatory for MDS_CONNECT RPCs. See also the discussion
-of inodes and extended attributes.
-[[mds-inode-bits-locks]]
+Lustre 1.4 and is mandatory for MDS_CONNECT RPCs.
[source,c]
----
A server reply uses the 'ocd_inodespace' value to inform the client of
the log base two of the size of an inode.
-Under some circumstances (for example when ZFS is the back end file
-system) there may be additional overhead in handling writes for each
-extent. The server uses the 'ocd_grant_extent' value to inform the
-client of the size in bytes consumed from its grant on the server when
-creating a new file. The client uses this value in calculating how
-much dirty write cache it has and whether it has reached the limit
-established by the target's grant.
+Under some circumstances (for example when the OSD is ZFS) there may
+be additional overhead in handling writes for each extent. The server
+uses the 'ocd_grant_extent' value to inform the client of the size in
+bytes consumed from its grant on the server when creating a new
+file. The client uses this value in calculating how much dirty write
+cache it has and whether it has reached the limit established by the
+target's grant.
If the OBD_CONNECT_TRANSNO flag is set then the 'ocd_transno' field is
honored. A server uses the 'ocd_transno' value during recovery to
Export
^^^^^^
-[[obd-export]]
+[[struct-obd-export]]
An 'obd_export' structure for a given target is created on a server
for each client that connects to that target. The exports for all the
The 'exp_handle' holds the cookie that the server generates at
*_CONNECT time to uniquely identify this connection from the client.
-This cookie is also sent back to the client in the reply and is
-then stored in the client's import.
+This cookie is also sent back to the client in the *_CONNECT reply and
+is then stored in the client's import.
//////////////////////////////////////////////////////////////////////
////vvvv
The 'exp_client_uuid' holds the UUID of the client connected to this
export. This UUID is randomly generated by the client and the same
-UUID is used by the client for connecting to all servers, so that
-the servers may identify the client amongst themselves if necessary.
+UUID is used by the client for connecting to all servers, so that the
+servers may identify the client amongst themselves if necessary. The
+client's UID appears in the *_CONNECT message (See
+<<ost-connect-rpc>>, <<mds-connect-rpc>>, and <<mgs-connect-rpc>>).
//////////////////////////////////////////////////////////////////////
////vvvv
the connection established between this target and the client this
export refers to. The 'exp_connect_data' describes the mutually
supported features that were negotiated between the client and server
-at connect time. See also the corresponding entry in the import and
-in the connect messages passed between the hosts.
+at connect time. See also the corresponding entry in the import
+(<<struct-obd-import>>) and the connect messages
+(<<ost-connect-rpc>>, <<mds-connect-rpc>>, and <mgs-connect-rpc>>)
+passed between the hosts.
//////////////////////////////////////////////////////////////////////
////vvvv
Import
^^^^^^
-[[obd-import]]
+[[struct-obd-import]]
The 'obd_import' structure holds the connection state for between each
client and each target it is connected to.
| LUSTRE_IMP_EVICTED | 10
|=====
+When the import is itself initialized it is set to
+LUSTRE_IMP_NEW. When a client initiates a *_CONNECT RPC it sets the
+state to LUSTRE_IMP_CONNECTING. Similarly, it sets the state to
+LUSTRE_IMP_DISCON as it initiates a *_DISCONNECT RPC. Reciving the
+reply to the *DISCONNECT RPC will set the state to
+LUSTRE_IMP_CLOSED. When a (successful) *_CONNECT RPC reply arrives the
+state is set to LUSTRE_IMP_FULL. If a target signals a problem or a
+recovery condition then the state will proceed through the replay and
+recover states. When the target signals that the client connection is
+invalid for some reaon the state will be set to
+LUSTRE_IMP_EVICTED. See <<eviction>> and <<recovery>>.
+
//////////////////////////////////////////////////////////////////////
////vvvv
fixme: what are the transitions between these states? The
////^^^^
//////////////////////////////////////////////////////////////////////
-The 'imp_remote_handle' is the handle sent by the target in a
-connection reply message to uniquely identify the export for this
-target and client that is maintained on the server. This is the handle
-used in all subsequent messages to the target.
+The 'imp_remote_handle' is the 'lustre_handle' sent by the target in a
+connection reply message (See <<struct-ptlrpc-body, specifically its
+'pb_handle' field, and <<ost-connect-rpc>> for OST_CONNECT). It
+uniquely identifies the target that this import represents. The client
+will set its 'pb_handle' value to this 'imp_remote_handle' in the
+descriptor for all subsequent RPC requests to this target (excluding
+other connect requests, of course).
//////////////////////////////////////////////////////////////////////
////vvvv
-OBD Statfs
-^^^^^^^^^^
+.OBD Statfs
[[struct-obd-statfs]]
-
-An 'obd_statfs' structure conveys file-system-wide information for the
-back-end file system of a given target (MDT or OST).
+****
+An 'obd_statfs' structure conveys file-system-wide information of a
+given target (MDT or OST).
[source,c]
----
__u32 os_spare9;
};
----
+****
Most of the fields correspond directly to the 'struct statfs' fields
returned by a 'statfs()' system call and have the same meaning. The
-values are for the back-end storage of the target in question (MDT or
-OST).
+values are for the target in question (MDT or OST).
-Teh 'os_type' field gives the type of the target's back-end file
-system:
+The 'os_type' field gives the kind of file system used for the
+target's OSD:
-.Back-end file types ('os_type') for Lustre targets
+.File system types ('os_type') for Lustre targets
[options="header"]
|====
| f_type | value
'os_bfree', and 'os_bavail' respectively. It does not necessarily
represent the minimum or optimal IO size.
-The 'os_namelen' field gives the maximum name length for files on the
-back-end file system in bytes.
+The 'os_namelen' field gives the maximum name length (in bytes) for
+files on the OSD.
The 'os_maxbytes' field is the maximum size of a single object
(i.e. the maximum byte offset that can be written to). This is the
same value as the 'ocd_maxbytes' field of the 'obd_connect_data'
structure.
-The 'os_state' field encodes the status of the underlying back-end
-file system. It can be:
+The 'os_state' field encodes the status of the OSD. It can be:
-.Back-end file system state
+.OSD state
[options="header"]
|====
| os_state flag | value
|====
In normal operation the 'os_state' value is returned as 0x0. If the
-back-end file system has a RAID configuration that is degraded or
+OSD has a RAID configuration that is degraded or
rebuilding the state is returned with the OS_STATE_DEGRADED (0x1) flag
set. If the file system has been set to read-only, either manually at
mount or automatcially due to detected corruption of the underlying
2*OST_MAX_PRECREATE (2*20000) on the number of precreates it will
allow for an OST. There are currently no precreated objects on an MDT
so it is just 0 in that case.
+
-Object Based Disk UUID
-^^^^^^^^^^^^^^^^^^^^^^
+.Object Based Disk UUID
[[struct-obd-uuid]]
-
+****
[source,c]
----
#define UUID_MAX 40
hexadecimal UUID of the form ''de305d54-75b4-431b-adb2-eb6b9e546014''
that is randomly generated. Servers may use a string-based identifier
of the form ''fsname-TGTindx_UUID''.
-
+****
-OST Body
-^^^^^^^^
+.OST Body
[[struct-ost-body]]
-
-The 'ost_body' structure just holds a 'struct 'obdo', which is where
+****
+The 'ost_body' structure just holds a 'struct obdo', which is where
all the actual information is conveyed.
[source,c]
struct obdo oa;
};
----
+****
-Obdo
-^^^^
+.Obdo
[[struct-obdo]]
-
+****
The 'obdo' structure conveys metadata about a resource on an OST.
[source,c]
The 'o_valid' field identifies which other fields in the structure are
to be interpreted. The flags are the same set (with additions) used
for the 'mdt_body' field 'mbo_valid' (see <<struct-mdt-body>>).
+****
[source,c]
----
#define OBD_MD_DEFAULT_MEA (0x0040000000000000ULL)
----
+****
The two fields here that are specific to the 'ost_body' case are the
'o_oi' and the 'o_stripe_idx', which give the identity of the OST and
the stripe index assigned to it.
+
+The 'o_handle' field is a 'lustre_handle' (See
+<<struct-lustre-handle>>). In this context it represents a lock. In
+most cases it will be 0. In an OST_PUNCH RPC (See <<truncate-rpcs>>) it
+may refer to a lock on the resource being modified that was previously
+acquired, though a lock is not required for that operation. The effect
+is to extend the timeout for that lock.
+****
-OST ID
-^^^^^^
+.OST ID
[[struct-ost-id]]
-
+****
The 'ost_id' identifies a single object on a particular OST.
[source,c]
The 'ost_id' structure contains an identifier for a single OST object.
The 'oi' structure holds the OST object identifier as used with Lustre
1.8 and earlier, where the 'oi_seq' field is typically zero, and the
-'oi_id' field is an integer identifying an object on a particular
-OST (which is identified separately). Since Lustre 2.5 it is possible
-for OST objects to also be identified with a unique FID that identifies
-both the OST on which it resides as well as the object identifier itself.
+'oi_id' field is an integer identifying an object on a particular OST
+(which is identified separately). Since Lustre 2.5 it is possible for
+OST objects to also be identified with a unique FID (see
+<<struct-lu-fid>>) that identifies both the OST on which it resides as
+well as the object identifier itself.
+****
-OST Lock Value Block
-^^^^^^^^^^^^^^^^^^^^
+.OST Lock Value Block
[[struct-ost-lvb]]
-
+****
The 'ost_lvb' structure is a "lock value block", and encompasses
attribute data for resources on the OST. It is returned from an OST to
a client requesting an extent lock. It is an optional part of an
__u32 lvb_padding;
};
----
+****
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[[struct-ptlrpc-body]]
-Every Lustre message starts with both the above header and an
-additional set of fields (in its first "buffer") given by the 'struct
-ptlrpc_body_v3' structure. This preamble has information information
-relevant to every RPC type. In particular, the RPC type is itself
-encoded in the 'pb_opc' Lustre operation number. The value of that
-opcode, as well as whether it is an RPC 'request' or 'reply',
-determines what else will be in the message following the preamble.
+Every Lustre message starts with a header (See <<struct-lustre-msg>>)
+describing a few generic details about the RPC, the most important of
+which is the number and sizes of additional "buffers" that will
+follow. The buffers just organize subsections of the RPC, and each
+comprises a set of fields. The fields of a buffer are presented here
+as 'struct' definitions because that is how they are actually
+organized, and it provides a convenient way to refer to groups of
+fields that appear together in many different RPCs. The first buffer
+in every RPC is given by the 'ptlrpc_body' structure, also known as
+the RPC descriptor. This descriptor identifies the type of the RPC via
+a Lustre operation code. The value of that 'opcode', as well as
+whether it is an RPC 'request' or 'reply', determines what other
+buffers will be in the message following the descriptor.
[source,c]
----
-#define PTLRPC_NUM_VERSIONS 4
-#define JOBSTATS_JOBID_SIZE 32
struct ptlrpc_body {
struct lustre_handle pb_handle;
__u32 pb_type;
__u32 pb_service_time;
__u32 pb_limit;
__u64 pb_slv;
- __u64 pb_pre_versions[PTLRPC_NUM_VERSIONS];
+ __u64 pb_pre_versions[4];
__u64 pb_padding[4];
- char pb_jobid[JOBSTATS_JOBID_SIZE];
+ char pb_jobid[32];
};
----
include::struct_lustre_handle.txt[]
-In a connection request, sent by a client to a server and regarding a
-specific target, the 'pb_handle' is 0. In the reply to a connection
-request, sent by the target, the handle is a value uniquely
-identifying the target. Subsequent messages between this client and
-this target will use this handle to to gain access to their shared
-state. The handle is persistent across client reconnects to the same
-instance of the server, but if the client unmounts the filesystem or
-is evicted then it must re-connect as a new client.
+The 'pb_handle' field of the RPC descriptor is a 'lustre_handle'. In a
+connection request RPC (MGS_CONNECT, MDS_CONNECT, or OST_CONNECT),
+sent by a client to a target, 'pb_handle' is 0. In the reply to that
+connection request 'pb_handle' is set to a new 'lustre_handle' that
+uniquely identifies the target (indeed, it uniquely identifies the
+'export', since each client gets a unique 'lustre_handle' for a given
+target). Subsequent request messages sent this client to this target
+will use that 'lustre_handle' (the 'pb_handle' field will be set to
+that value) to to gain access to their shared state. In subsequent RPC
+reply messages (after teh *_CONNECT reply) the 'pb_handle' field is
+0. The 'lustre_handle' is persistent across client reconnects to the
+same instance of the server, but if the client unmounts the filesystem
+or is evicted then it must re-connect as a new client, with a new
+'lustre_handle'.
The 'pb_type' is PTL_RPC_MSG_REQUEST in messages when they are
initiated, it is PTL_RPC_MSG_REPLY in a reply, and it is
such as those that emerge from processing the actual message content,
do not use the PTL_RPC_MSG_ERR type.
-The 'pb_status' field provides an error return code, if any, for the
-RPC. When 'pb_type' = PTL_RPC_MSG_ERR the 'pb_status' will also be
-set to one of the following message handling errors:
+The 'pb_status' field provides an error return code for the RPC. When
+'pb_type' = PTL_RPC_MSG_ERR the 'pb_status' will also be set to one of
+the following message handling errors:
-.format
+.'pb_type' = PTL_RPC_MSG_ERR status return codes
[options="header"]
|====
-| pb_type | pb_status | if
-| PTL_RPC_MSG_ERR | -ENOMEM | No memory for reply buffer
-| PTL_RPC_MSG_ERR | -ENOTSUPP | Invalid opcode
-| PTL_RPC_MSG_ERR | -EINVAL | Bad magic or version
-| PTL_RPC_MSG_ERR | -EPROTO | Request is malformed or cannot be
- processed in current context
+| pb_status | if
+| -ENOMEM | No memory for reply buffer
+| -ENOTSUPP | Invalid opcode
+| -EINVAL | Bad magic or version
+| -EPROTO | Request is malformed or cannot be
+ processed in current context
|====
A PTL_RPC_MSG_ERR message does not need to allocate memory, so it
should normally be sent as a reply even if there is not enough memory
-to allocate the normal reply buffer, unless the underlying network
-transport itself cannot allocate memory to send it. (fixme: and what
-happens then?)
+to allocate the normal reply buffer.
In most cases there is a reply with 'pb_type' = PTL_RPC_MSG_REPLY,
indicating that the request was processed, but it may still have
will be documented with the respective RPCs, and those that are more
generic are listed here:
-.format
+.'pb_type' = PTL_RPC_MSG_REPLY status return codes
[options="header"]
|====
-| pb_type | pb_status | meaning
-| PTL_RPC_MSG_REPLY | -ENOTCONN | Client is not connected to the
- target, typically meaning the
- server was restarted or the
- client was evicted, and the
- client needs to reconnect.
-| PTL_RPC_MSG_REPLY | -EINPROGRESS | The request cannot be processed
- currently due to some other
- factor, such as during initial
- mount, a delay contacting the
- quota master during a write, or
- LFSCK rebuilding the OI table,
- but the client should continue
- to retry after a delay until
- interrupted or successful. This
- avoids blocking the server
- threads with client requests
- that cannot currently be
- processed, but other requests
- might be processed in the
- meantime.
-| PTL_RPC_MSG_REPLY | -ESHUTDOWN | The server is being stopped and
- no new connections are allowed.
+| pb_status | meaning
+| -ENOTCONN | Client is not connected to the
+ target, typically meaning the
+ server was restarted or the
+ client was evicted, and the
+ client needs to reconnect.
+| -EINPROGRESS | The request cannot be processed
+ currently due to some other
+ factor, such as during initial
+ mount, a delay contacting the
+ quota master during a write, or
+ LFSCK rebuilding the OI table,
+ but the client should continue
+ to retry after a delay until
+ interrupted or successful. This
+ avoids blocking the server
+ threads with client requests
+ that cannot currently be
+ processed, but other requests
+ might be processed in the
+ meantime.
+| -ESHUTDOWN | The server is being stopped and
+ no new connections are allowed.
|====
-The significance of -ENOTCONN is discussed more fully in
-<<connection>>, but a brief comment may be useful here. The networking
-layers supporting the exchange of RPCs can be in good working order
-when 'pb_status' = -ENOTCONN is returned in an RPC reply message. The
-connection refered to by that status is the Lustre connection. That
-connection is part of the shared state between Lustre clients and
-servers that gets established via *_CONNECT RPCs,
-and can be lost due to an 'eviction' in the face of temporary connection
-failure or in case of an unresponsive client (from the server's point
-of view). So, even when that Lusre connection is lost (or has not been
-established, yet), RPC messages such as *_CONNECT can be exchanged.
+A PtlRPC reply message with 'pb_status' = -ENOTCONN can be returned
+for any type of PtlRPC request message other than one of MGS_CONNECT,
+MDS_CONNECT, or OST_CONNECT (those RPCs establish connections).
+-ENOTCONN indicates that the server, the recipient of the request
+message, does not have a valid connection for the client. This can
+come about if a client has been evicted, or if, after a server
+reboots, the client failed to reestablish its connection during
+recovery. Establishing a connection is discussed more fully in
+<<connection>>, eviction in <<eviction>>, and recovery in
+<<recovery>>.
The 'pb_version' identifies the version of the Lustre protocol and is
derived from the following constants. The lower two bytes give the
Lustre operation (number 38). The following list gives the name used
and the value for each operation.
-[source,c]
-----
-typedef enum {
- OST_REPLY = 0,
- OST_GETATTR = 1,
- <<ost-setattr-rpc,OST_SETATTR>> = 2,
- OST_READ = 3,
- OST_WRITE = 4,
- OST_CREATE = 5,
- OST_DESTROY = 6,
- OST_GET_INFO = 7,
- <<ost-connect-rpc,OST_CONNECT>> = 8,
- <<ost-connect-rpc,OST_DISCONNECT>> = 9,
- OST_PUNCH = 10,
- OST_OPEN = 11,
- OST_CLOSE = 12,
- OST_STATFS = 13,
- OST_SYNC = 16,
- OST_SET_INFO = 17,
- OST_QUOTACHECK = 18,
- OST_QUOTACTL = 19,
- OST_QUOTA_ADJUST_QUNIT = 20,
-
- <<mds-getattr-rpc,MDS_GETATTR>> = 33,
- MDS_GETATTR_NAME = 34,
- MDS_CLOSE = 35,
- MDS_REINT = 36,
- MDS_READPAGE = 37,
- <<mds-connect-rpc,MDS_CONNECT>> = 38,
- <<mds-disconnect,MDS_DISCONNECT>> = 39,
- <<mds-getstatus-rpc,MDS_GETSTATUS>> = 40,
- <mds-statfs-rpc,MDS_STATFS>> = 41,
- MDS_PIN = 42,
- MDS_UNPIN = 43,
- MDS_SYNC = 44,
- MDS_DONE_WRITING = 45,
- MDS_SET_INFO = 46,
- MDS_QUOTACHECK = 47,
- MDS_QUOTACTL = 48,
- <<mds-getxattr-rpc,MDS_GETXATTR>> = 49,
- MDS_SETXATTR = 50,
- MDS_WRITEPAGE = 51,
- MDS_IS_SUBDIR = 52,
- MDS_GET_INFO = 53,
- MDS_HSM_STATE_GET = 54,
- MDS_HSM_STATE_SET = 55,
- MDS_HSM_ACTION = 56,
- MDS_HSM_PROGRESS = 57,
- MDS_HSM_REQUEST = 58,
- MDS_HSM_CT_REGISTER = 59,
- MDS_HSM_CT_UNREGISTER = 60,
- MDS_SWAP_LAYOUTS = 61,
-
- <<ldlm-enqueue-rpc,LDLM_ENQUEUE>> = 101,
- LDLM_CONVERT = 102,
- <ldlm-cancel-rpc,LDLM_CANCEL>> = 103,
- <ldlm_bl_callback-rpc,LDLM_BL_CALLBACK>> = 104,
- <ldlm-cp-callback-rpc,LDLM_CP_CALLBACK>> = 105,
- LDLM_GL_CALLBACK = 106,
- LDLM_SET_INFO = 107,
-
- <<mgs-connect-rpc,MGS_CONNECT>> = 250,
- <<mgs-disconnect-rpc,MGS_DISCONNECT>> = 251,
- MGS_EXCEPTION = 252,
- MGS_TARGET_REG = 253,
- MGS_TARGET_DEL = 254,
- MGS_SET_INFO = 255,
- <<mgs-config-read-rpc,MGS_CONFIG_READ>> = 256,
-
- OBD_PING = 400,
- OBD_LOG_CANCEL = 401,
- OBD_QC_CALLBACK = 402,
- OBD_IDX_READ = 403,
-
- <<llog-origin-handle-create-rpc,LLOG_ORIGIN_HANDLE_CREATE>> = 501,
- <<llog-origin-handle-next-block,LLOG_ORIGIN_HANDLE_NEXT_BLOCK>> = 502,
- <<llog-origin-handle-read-header,LLOG_ORIGIN_HANDLE_READ_HEADER>> = 503,
- LLOG_ORIGIN_HANDLE_WRITE_REC = 504,
- LLOG_ORIGIN_HANDLE_CLOSE = 505,
- LLOG_ORIGIN_CONNECT = 506,
- LLOG_ORIGIN_HANDLE_PREV_BLOCK = 508,
- LLOG_ORIGIN_HANDLE_DESTROY = 509,
-
- QUOTA_DQACQ = 601,
- QUOTA_DQREL = 602,
-
- SEQ_QUERY = 700,
-
- SEC_CTX_INIT = 801,
- SEC_CTX_INIT_CONT = 802,
- SEC_CTX_FINI = 803,
-
- FLD_QUERY = 900,
- FLD_READ = 901,
-
- UPDATE_OBJ = 1000
-} cmd_t;
-----
-The symbols and values above identify the operations Lustre uses in
-its protocol. They are examined in detail in the
-<<lustre-prcs,Lustre RPCs>> section. Lustre carries out
-each of these operations via the exchange of a pair of messages: a
-request and a reply.
+.Lustre Operation Codes ("opcodes")
+[options="header"]
+|====
+| name | 'pb_opc'
+| OST_REPLY | 0
+| OST_GETATTR | 1
+| <<ost-setattr-rpc>> | 2
+| OST_READ | 3
+| OST_WRITE | 4
+| OST_CREATE | 5
+| OST_DESTROY | 6
+| OST_GET_INFO | 7
+| <<ost-connect-rpc>> | 8
+| <<ost-disconnect-rpc>> | 9
+| OST_PUNCH | 10
+| OST_OPEN | 11
+| OST_CLOSE | 12
+| OST_STATFS | 13
+| OST_SYNC | 16
+| OST_SET_INFO | 17
+| OST_QUOTACHECK | 18
+| OST_QUOTACTL | 19
+| OST_QUOTA_ADJUST_QUNIT | 20
+| |
+| <<mds-getattr-rpc>> | 33
+| MDS_GETATTR_NAME | 34
+| MDS_CLOSE | 35
+| MDS_REINT | 36
+| MDS_READPAGE | 37
+| <<mds-connect-rpc>> | 38
+| <<mds-disconnect-rpc>> | 39
+| <<mds-getstatus-rpc>> | 40
+| <mds-statfs-rpc>> | 41
+| MDS_PIN | 42
+| MDS_UNPIN | 43
+| MDS_SYNC | 44
+| MDS_DONE_WRITING | 45
+| MDS_SET_INFO | 46
+| MDS_QUOTACHECK | 47
+| MDS_QUOTACTL | 48
+| <<mds-getxattr-rpc>> | 49
+| MDS_SETXATTR | 50
+| MDS_WRITEPAGE | 51
+| MDS_IS_SUBDIR | 52
+| MDS_GET_INFO | 53
+| MDS_HSM_STATE_GET | 54
+| MDS_HSM_STATE_SET | 55
+| MDS_HSM_ACTION | 56
+| MDS_HSM_PROGRESS | 57
+| MDS_HSM_REQUEST | 58
+| MDS_HSM_CT_REGISTER | 59
+| MDS_HSM_CT_UNREGISTER | 60
+| MDS_SWAP_LAYOUTS | 61
+| |
+| <<ldlm-enqueue-rpc>> | 101
+| LDLM_CONVERT | 102
+| <<ldlm-cancel-rpc>> | 103
+| <<ldlm-bl-callback-rpc>> | 104
+| <<ldlm-cp-callback-rpc>> | 105
+| LDLM_GL_CALLBACK | 106
+| LDLM_SET_INFO | 107
+| |
+| <<mgs-connect-rpc>> | 250
+| <<mgs-disconnect-rpc>> | 251
+| MGS_EXCEPTION | 252
+| MGS_TARGET_REG | 253
+| MGS_TARGET_DEL | 254
+| MGS_SET_INFO | 255
+| <<mgs-config-read-rpc>> | 256
+| |
+| OBD_PING | 400
+| OBD_LOG_CANCEL | 401
+| OBD_QC_CALLBACK | 402
+| OBD_IDX_READ | 403
+| |
+| <<llog-origin-handle-create-rpc>> | 501
+| <<llog-origin-handle-next-block-rpc>> | 502
+| <<llog-origin-handle-read-header-rpc>> | 503
+| LLOG_ORIGIN_HANDLE_WRITE_REC | 504
+| LLOG_ORIGIN_HANDLE_CLOSE | 505
+| LLOG_ORIGIN_CONNECT | 506
+| LLOG_ORIGIN_HANDLE_PREV_BLOCK | 508
+| LLOG_ORIGIN_HANDLE_DESTROY | 509
+| |
+| QUOTA_DQACQ | 601
+| QUOTA_DQREL | 602
+| |
+| SEQ_QUERY | 700
+| |
+| SEC_CTX_INIT | 801
+| SEC_CTX_INIT_CONT | 802
+| SEC_CTX_FINI | 803
+| |
+| FLD_QUERY | 900
+| FLD_READ | 901
+| |
+| UPDATE_OBJ | 1000
+|====
The 'pb_status' field was already mentioned above in conjuction with
the 'pb_type' field in replies. In a request message 'pb_status' is
requested operation. When an error is being reported the value will
encode a standard Linux kernel (POSIX) error code as initially
defined for the i386/x86_64 architecture. The 'pb_status' value is
-returned as a negative number, so for example, a permissions error
+returned as a negative number. For example, a permissions error
would be indicated as -EPERM.
'pb_last_xid' and 'pb_last_seen' are not used.
The 'pb_last_committed' value is always zero in a request. In a reply
it is the highest transaction number that has been committed to
-storage. The transaction numbers are maintained on a per-target basis
-and each series of transaction numbers is a strictly increasing
-sequence for modifications originating from any client. This field is
-set in any kind of reply message including pings and non-modifying
-transactions. If 'pb_last_committed' is larger than, or equal to, any
-of the client's uncommitted requests (see 'pb_transno' below) then the
-server is confirming those requests have been committed to stable
-storage. At that point the client may free those request structures
-as they will no longer be necessary for replay during recovery.
-
-The 'pb_transno' value is always zero in a new request. It is also
-zero for replies to operations that do not modify the file system. For
-replies to operations that do modify the file system it is the
-target-unique, server-assigned transaction number for the client
-request. The 'pb_transno' assigned to each modifying request is in
-strictly increasing order, but may not be sequential for a single
-client, and the client may receive replies in a different order than
-they were processed by the server.Upon receipt of the reply, the
-client copies this transaction number from 'pb_transno' of the reply
-to 'pb_transno' of the saved request. If 'pb_transno' is larger than
-'pb_last_commited' (above) then the request has only been processed at
-the target and is not yet committed to stable storage. The client
-must save the request for later resend to the server in case the
-target fails before the modification can be committed to disk.If the
-request has to be replayed it will include the transaction number.
-
-The 'pb_flags' value governs the client state machine. Fixme: document
-what the states and transitions are of this state machine. Currently,
-only the bottom two bytes are used, and they encode state according to
-the following values:
+storage. See <<transno>>. This field is set in any kind of reply
+message including pings and non-modifying transactions. If
+'pb_last_committed' is larger than, or equal to, any of the client's
+uncommitted requests (see 'pb_transno' below) then the server is
+confirming those requests have been committed to stable storage. At
+that point the client may free those request structures as they will
+no longer be necessary for replay during recovery.
+
+The 'pb_transno' value is zero in a new request. It is also zero for
+replies to operations that do not modify the file system. For replies
+to operations that do modify the file system it is the target-unique,
+server-assigned transaction number for the client request. See
+<<transno>>. Upon receipt of the reply, the client copies this
+transaction number from 'pb_transno' of the reply to 'pb_transno' of
+the saved request. If 'pb_transno' is larger than 'pb_last_commited'
+(above) then the request has been processed at the target but is not
+yet committed to stable storage. The client must save the request for
+later resend to the server in case the target fails before the
+modification can be committed to disk. If the request has to be
+replayed it will include the transaction number.
+
+The 'pb_flags' value governs the client state machine. Currently, only
+the least significant two bytes are used. The field encodes state
+according to the following values:
[source,c]
----
MGS_DELAY_REPLAY is currently unused.
-MSG_VERSION_REPLAY indicates that a replayed request has
-pb_pre_versions[] filled with the prior object versions and can be
-used with Version Based Recovery.
+MSG_VERSION_REPLAY indicates that a replayed request has the
+'pb_pre_versions' array filled with the prior object versions and can
+be used with Version Based Recovery.
MSG_LOCK_REPLAY_DONE indicates the client has completed lock replay,
and is ready to finish recovery.
#define MSG_CONNECT_TRANSNO 0x00000100
----
-MGS_CONNECT_RECOVERING indicate the server is in recovery
+MSG_CONNECT_RECOVERING indicates the server is in recovery
-MGS_CONNECT_RECONNECT indicates the client is reconnecting after
+MSG_CONNECT_RECONNECT indicates the client is reconnecting after
non-responsiveness from the server.
-MGS_CONNECT_REPLAYABLE indicates the server connection supports RPC
+MSG_CONNECT_REPLAYABLE indicates the server connection supports RPC
replay (only OSTs support non-recoverable connections, but that is not
the default).
-The MGS_CONNECT_LIBCLIENT is for the a 'liblustre' client. It is
+The MSG_CONNECT_LIBCLIENT is for the a 'liblustre' client. It is
currently unused.
-The client sends MGS_CONNECT_INITIAL the first time the client is
+The client sends MSG_CONNECT_INITIAL the first time the client is
connecting to the server. MSG_CONNECT_INITIAL connections are not
allowed during server recovery.
-MGS_CONNECT_ASYNC is currently unused.
+MSG_CONNECT_ASYNC is currently unused.
MSG_CONNECT_NEXT_VER indicates that the client can understand the next
higher protocol version in addition to the currently used protocol, and
The 'pb_conn_cnt' (connection count) value in a request message
reports the client's "era", which is part of the client and server's
-shared state. The value of the era is initialized to one when it is
-first connected to the MDT. Each subsequent connection (after an
+shared state. The value of the era is initialized to 1 when it is
+first connected to the target. Each subsequent connection (after an
eviction) increments the era for the client. Since the 'pb_conn_cnt'
reflects the client's era at the time the message was composed the
server can use this value to discard late-arriving messages requesting
associated with the given queue, not just the specific message that
initiated the request. Finally, in a reply message (one that does
indicate the operation has been initiated) the timeout value updates
-the timeout interval for the queue. Is this last point different from
-the "early reply" update?
+the timeout interval for the queue.
The 'pb_service_time' value is zero in a request. In a reply it
indicates how long this particular operation actually took from the
--- /dev/null
+Target
+~~~~~~
+[[target]]
+
+Lustre servers manage permanent storage on behalf of the clients. That
+storage is divided into logical resources for management data,
+metadata (namespace and index data), and object storage. Each such
+storage resource is called a target. Managment data is stored on a
+single management server (MGS) with a single target, also called the
+MGS. Namespace and index data are maintained on one or more metadata
+servers (MDSs), and each MDS may have several targets (MDTs). Object
+storage is maintained on object servers (OSSs), and each OSS can have
+several targets (OSTs), as well. Each target is based on an Object
+Storage Device (OSD) which can be either an ldikfs-based file system
+or a zfs-based file system.
+
+
~~~~~~~~~~~~~~~~~~
[[transno]]
-For each target there is a sequence of values (a strictly increasing
-series of numbers) where each operation that can modify the file
-system is assigned the next number in the series. This is the
-transaction number, and it imposes a strict serial ordering for all
-file system modifying operations. For file system modifying
-requests the server assigns the next value in the sequence and
-informs the client of the value in the 'pb_transno' field of the
-'ptlrpc_body' of its reply to the client's request. For replys to
-requests that do not modify the file system the 'pb_transno' field in
-the 'ptlrpc_body' is just set to 0.
-
+Every transaction that modifies a target is assigned a 'transaction
+number'. The transaction numbers for a given target are assigned in a
+strictly increasing sequence. In this way, the transactions from all
+clients are strictly ordered. The server assigns the next value in
+the target's sequence and informs the client of the value in the
+'pb_transno' field of the 'ptlrpc_body' of its reply to the client's
+request. See <<struct-ptlrpc-body>>.
6 <-MGS_DISCONNECT
//////////////////////////////////////////////////////////////////////
+*1 - Client issues an OST_DICONNECT to each OST.*
+
+.OST_DISCONNECT Request Packet Structure
+image::ost-disconnect-request.png["OST_DISCONNECT Request Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ost-disconnect-request.png diagram resembles this text art:
+
+ OST_DISCONNECT:
+ --request------
+ | ptlrpc_body |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
+See <<ost-disconnect-rpc>>.
+
+*2 - Each OST replies to the OST_DISCONNECT.*
+
+.OST_DISCONNECT Reply Packet Structure
+image::ost-disconnect-reply.png["OST_DISCONNECT Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The ost-disconnect-reply.png diagram resembles this text art:
+
+ OST_DISCONNECT:
+ --reply--------
+ | ptlrpc_body |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
+*3 - Client issues an MDS_DICONNECT to each MDT.*
+
+.MDS_DISCONNECT Request Packet Structure
+image::mds-disconnect-request.png["MDS_DISCONNECT Request Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-disconnect-request.png diagram resembles this text art:
+
+ MDS_DISCONNECT:
+ --request------
+ | ptlrpc_body |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
+See <<mds-disconnect-rpc>>.
+
+*4 - Each MDT replies to the MDS_DISCONNECT.*
+
+.MDS_DISCONNECT Reply Packet Structure
+image::mds-disconnect-reply.png["MDS_DISCONNECT Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mds-disconnect-reply.png diagram resembles this text art:
+
+ MDS_DISCONNECT:
+ --reply--------
+ | ptlrpc_body |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
+*5 - Client issues an MGS_DICONNECT to the MGS.*
+
+.MGS_DISCONNECT Request Packet Structure
+image::mgs-disconnect-request.png["MGS_DISCONNECT Request Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mgs-disconnect-request.png diagram resembles this text art:
+
+ MGS_DISCONNECT:
+ --request------
+ | ptlrpc_body |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
+See <<mgs-disconnect-rpc>>.
+
+*6 - The MGS replies to the MGS_DISCONNECT.*
+
+.MGS_DISCONNECT Reply Packet Structure
+image::mgs-disconnect-reply.png["MGS_DISCONNECT Reply Packet Structure",height=50]
+
+//////////////////////////////////////////////////////////////////////
+The mgs-disconnect-reply.png diagram resembles this text art:
+
+ MGS_DISCONNECT:
+ --reply--------
+ | ptlrpc_body |
+ ---------------
+//////////////////////////////////////////////////////////////////////
+
git://git.whamcloud.com / doc / protocol.git / commitdiff