--- /dev/null
+OBD Connect Data
+^^^^^^^^^^^^^^^^
+[[struct-obd-connect-data]]
+
+An 'obd_connect_data' structure accompanies every connect operation in
+both the request message and in the reply message. It contains the
+mutually supported features that are negotiated between the client and
+server at *_CONNECT time.
+
+At *_CONNECT time the client sends in its request 'ocd_connect_flags'
+the flags for all features that it understands and intends to use (for
+example 'OBD_CONNECT_RDONLY' is optional depending on client mount
+options). The request also contains other fields that are only valid
+if the matching flag is set. The server replies in 'ocd_connect_flags'
+with the subset of feature flags that it understands and intends to honour.
+The server may set fields in the reply for mutually-understood features.
+
+[source,c]
+----
+struct obd_connect_data {
+ __u64 ocd_connect_flags;
+ __u32 ocd_version; /* OBD_CONNECT_VERSION */
+ __u32 ocd_grant; /* OBD_CONNECT_GRANT */
+ __u32 ocd_index; /* OBD_CONNECT_INDEX */
+ __u32 ocd_brw_size; /* OBD_CONNECT_BRW_SIZE */
+ __u64 ocd_ibits_known; /* OBD_CONNECT_IBITS */
+ __u8 ocd_blocksize; /* OBD_CONNECT_GRANT_PARAM */
+ __u8 ocd_inodespace; /* OBD_CONNECT_GRANT_PARAM */
+ __u16 ocd_grant_extent; /* OBD_CONNECT_GRANT_PARAM */
+ __u32 ocd_unused;
+ __u64 ocd_transno; /* OBD_CONNECT_TRANSNO */
+ __u32 ocd_group; /* OBD_CONNECT_MDS */
+ __u32 ocd_cksum_types; /* OBD_CONNECT_CKSUM */
+ __u32 ocd_max_easize; /* OBD_CONNECT_MAX_EASIZE */
+ __u32 ocd_instance;
+ __u64 ocd_maxbytes; /* OBD_CONNECT_MAXBYTES */
+ __u64 padding1;
+ __u64 padding2;
+ __u64 padding3;
+ __u64 padding4;
+ __u64 padding5;
+ __u64 padding6;
+ __u64 padding7;
+ __u64 padding8;
+ __u64 padding9;
+ __u64 paddingA;
+ __u64 paddingB;
+ __u64 paddingC;
+ __u64 paddingD;
+ __u64 paddingE;
+ __u64 paddingF;
+};
+----
+
+The 'ocd_connect_flags' field encodes the connect flags giving the
+capabilities of a connection between client and target. Several of
+those flags (noted in comments above and the discussion below)
+actually control whether the remaining fields of 'obd_connect_data'
+get used. The [[obd-connect-flags]] flags are:
+
+[source,c]
+----
+#define OBD_CONNECT_RDONLY 0x1ULL /*client has read-only access*/
+#define OBD_CONNECT_INDEX 0x2ULL /*connect specific LOV idx */
+#define OBD_CONNECT_MDS 0x4ULL /*connect from MDT to OST */
+#define OBD_CONNECT_GRANT 0x8ULL /*OSC gets grant at connect */
+#define OBD_CONNECT_SRVLOCK 0x10ULL /*server takes locks for cli */
+#define OBD_CONNECT_VERSION 0x20ULL /*Lustre versions in ocd */
+#define OBD_CONNECT_REQPORTAL 0x40ULL /*Separate non-IO req portal */
+#define OBD_CONNECT_ACL 0x80ULL /*access control lists */
+#define OBD_CONNECT_XATTR 0x100ULL /*client use extended attr */
+#define OBD_CONNECT_TRUNCLOCK 0x400ULL /*locks on server for punch */
+#define OBD_CONNECT_TRANSNO 0x800ULL /*replay sends init transno */
+#define OBD_CONNECT_IBITS 0x1000ULL /*support for inodebits locks*/
+#define OBD_CONNECT_ATTRFID 0x4000ULL /*Server can GetAttr By Fid*/
+#define OBD_CONNECT_NODEVOH 0x8000ULL /*No open hndl on specl nodes*/
+#define OBD_CONNECT_RMT_CLIENT 0x10000ULL /*Remote client */
+#define OBD_CONNECT_RMT_CLIENT_FORCE 0x20000ULL /*Remote client by force */
+#define OBD_CONNECT_BRW_SIZE 0x40000ULL /*Max bytes per rpc */
+#define OBD_CONNECT_QUOTA64 0x80000ULL /*Not used since 2.4 */
+#define OBD_CONNECT_CANCELSET 0x400000ULL /*Early batched cancels. */
+#define OBD_CONNECT_SOM 0x800000ULL /*Size on MDS */
+#define OBD_CONNECT_AT 0x1000000ULL /*client uses AT */
+#define OBD_CONNECT_LRU_RESIZE 0x2000000ULL /*LRU resize feature. */
+#define OBD_CONNECT_MDS_MDS 0x4000000ULL /*MDS-MDS connection */
+#define OBD_CONNECT_REAL 0x8000000ULL /*real connection */
+#define OBD_CONNECT_CHANGE_QS 0x10000000ULL /*Not used since 2.4 */
+#define OBD_CONNECT_CKSUM 0x20000000ULL /*support several cksum algos*/
+#define OBD_CONNECT_FID 0x40000000ULL /*FID is supported by server */
+#define OBD_CONNECT_VBR 0x80000000ULL /*version based recovery */
+#define OBD_CONNECT_LOV_V3 0x100000000ULL /*client supports LOV v3 EA */
+#define OBD_CONNECT_GRANT_SHRINK 0x200000000ULL /* support grant shrink */
+#define OBD_CONNECT_SKIP_ORPHAN 0x400000000ULL /* don't reuse orphan objids */
+#define OBD_CONNECT_MAX_EASIZE 0x800000000ULL /* preserved for large EA */
+#define OBD_CONNECT_FULL20 0x1000000000ULL /* it is 2.0 client */
+#define OBD_CONNECT_LAYOUTLOCK 0x2000000000ULL /* client uses layout lock */
+#define OBD_CONNECT_64BITHASH 0x4000000000ULL /* client supports 64-bits
+ * directory hash */
+#define OBD_CONNECT_MAXBYTES 0x8000000000ULL /* max stripe size */
+#define OBD_CONNECT_IMP_RECOV 0x10000000000ULL /* imp recovery support */
+#define OBD_CONNECT_JOBSTATS 0x20000000000ULL /* jobid in ptlrpc_body */
+#define OBD_CONNECT_UMASK 0x40000000000ULL /* create uses client umask */
+#define OBD_CONNECT_EINPROGRESS 0x80000000000ULL /* client handles -EINPROGRESS
+ * RPC error properly */
+#define OBD_CONNECT_GRANT_PARAM 0x100000000000ULL/* extra grant params used for
+ * finer space reservation */
+#define OBD_CONNECT_FLOCK_OWNER 0x200000000000ULL /* for the fixed 1.8
+ * policy and 2.x server */
+#define OBD_CONNECT_LVB_TYPE 0x400000000000ULL /* variable type of LVB */
+#define OBD_CONNECT_NANOSEC_TIME 0x800000000000ULL /* nanosecond timestamps */
+#define OBD_CONNECT_LIGHTWEIGHT 0x1000000000000ULL/* lightweight connection */
+#define OBD_CONNECT_SHORTIO 0x2000000000000ULL/* short io */
+#define OBD_CONNECT_PINGLESS 0x4000000000000ULL/* pings not required */
+#define OBD_CONNECT_FLOCK_DEAD 0x8000000000000ULL/* deadlock detection */
+#define OBD_CONNECT_DISP_STRIPE 0x10000000000000ULL/* create stripe disposition*/
+#define OBD_CONNECT_OPEN_BY_FID 0x20000000000000ULL /* open by fid won't pack
+ name in request */
+----
+
+Each flag corresponds to a particular capability that the client and
+target together will honor. A client will send a message including
+some subset of these capabilities during a connection request to a
+specific target. This tells the server what capabilities it has. The
+server then replies with the subset of those capabilities it agrees to
+honor (for the given target).
+
+If the 'OBD_CONNECT_VERSION' flag is set then the 'ocd_version' field is
+valid. The 'ocd_version' gives an encoding of the Lustre
+version. For example, Version 2.7.55 would be hexadecimal number
+0x02075500.
+
+If the OBD_CONNECT_GRANT flag is set then the 'ocd_grant' field is
+valid. The 'ocd_grant' value in a reply (to a connection request)
+sets the client's grant.
+
+If the OBD_CONNECT_INDEX flag is set then the 'ocd_index' field is
+valid. The 'ocd_index' value is set in a request to hold the LOV
+index of the OST or the LMV index of the MDT. The server should
+refuse connections to targets for which the 'ocd_index' does not
+match the actual target index.
+
+If the OBD_CONNECT_BRW_SIZE flag is set then the 'ocd_brw_size' field
+is valid. The 'ocd_brw_size' value sets the maximum supported bulk
+transfer size. The client proposes a value in its connection request,
+and the server's reply will either accept the requested size or
+further limit the size. The server will not increase the client's
+requested maximum bulk transfer size.
+
+If the OBD_CONNECT_IBITS flag is set then the 'ocd_ibits_known' field
+is valid. The 'ocd_ibits_known' flags determine the handling of
+locks on MDS inodes. The OBD_CONNECT_IBITS flag was introduced in
+Lustre 1.4 and is mandatory for MDS_CONNECT RPCs. See also the discussion
+of inodes and extended attributes.
+[[mds-inode-bits-locks]]
+
+[source,c]
+----
+#define MDS_INODELOCK_LOOKUP 0x000001 /* For namespace, dentry etc, and also
+ * was used to protect permission (mode,
+ * owner, group etc) before 2.4. */
+#define MDS_INODELOCK_UPDATE 0x000002 /* size, links, timestamps */
+#define MDS_INODELOCK_OPEN 0x000004 /* For opened files */
+#define MDS_INODELOCK_LAYOUT 0x000008 /* for layout */
+#define MDS_INODELOCK_PERM 0x000010 /* separate permission bits */
+#define MDS_INODELOCK_XATTR 0x000020 /* extended attributes */
+----
+
+//////////////////////////////////////////////////////////////////////
+/* The PERM bit is added int 2.4, and it is used to protect permission(mode,
+ * owner, group, acl etc), so to separate the permission from LOOKUP lock.
+ * Because for remote directories(in DNE), these locks will be granted by
+ * different MDTs(different ldlm namespace).
+ *
+ * For local directory, MDT will always grant UPDATE_LOCK|PERM_LOCK together.
+ * For Remote directory, the master MDT, where the remote directory is, will
+ * grant UPDATE_LOCK|PERM_LOCK, and the remote MDT, where the name entry is,
+ * will grant LOOKUP_LOCK. */
+#define MDS_INODELOCK_PERM 0x000010
+#define MDS_INODELOCK_XATTR 0x000020 /* extended attributes */
+//////////////////////////////////////////////////////////////////////
+
+If the OBD_CONNECT_GRANT_PARAM flag is set then the 'ocd_blocksize',
+'ocd_inodespace', and 'ocd_grant_extent' fields are honored. A server
+reply uses the 'ocd_blocksize' value to inform the client of the log
+base two of the size in bytes of the backend file system's blocks.
+
+A server reply uses the 'ocd_inodespace' value to inform the client of
+the log base two of the size of an inode.
+
+Under some circumstances (for example when ZFS is the back end file
+system) there may be additional overhead in handling writes for each
+extent. The server uses the 'ocd_grant_extent' value to inform the
+client of the size in bytes consumed from its grant on the server when
+creating a new file. The client uses this value in calculating how
+much dirty write cache it has and whether it has reached the limit
+established by the target's grant.
+
+If the OBD_CONNECT_TRANSNO flag is set then the 'ocd_transno' field is
+honored. A server uses the 'ocd_transno' value during recovery to
+inform the client of the transaction number at which it should begin
+replay.
+
+If the OBD_CONNECT_MDS flag is set then the 'ocd_group' field is
+valid. When an MDT connects to an OST the 'ocd_group' field informs
+the OSS of the MDT's index. Objects on that OST for that MDT will be
+in a common namespace served by that MDT.
+
+If the OBD_CONNECT_CKSUM flag is set then the 'ocd_cksum_types' field
+is valid. The client uses the 'ocd_cksum_types' field to propose
+to the server the client's available (possibly hardware assisted)
+checksum mechanisms. The server replies with the checksum types it has
+available and that are most efficient on the server. The client may
+employ any of the replied checksum algorithms for a given bulk transfer,
+but will typically select the fastest of the agreed algorithms.
+
+If the OBD_CONNECT_MAX_EASIZE flag is set then the 'ocd_max_easize'
+field is valid. The server uses 'ocd_max_easize' to inform the
+client about the amount of space that can be allocated in each inode
+for extended attributes. The 'ocd_max_easize' specifically refers to
+the space used for layout information. This allows the client to
+determine the maximum layout size (and hence stripe count) that can be
+stored on the MDT.
+
+The 'ocd_instance' field (alone) is not governed by an OBD_CONNECT_*
+flag. The MGS uses the 'ocd_instance' value in its reply to a
+connection request to inform the server and target of the "era" of its
+connection. The MGS initializes the era value for each server to zero
+and increments that value every time the target connects. This
+supports imperative recovery.
+
+If the OBD_CONNECT_MAXBYTES flag is set then the 'ocd_maxbytes' field
+is honored. An OSS uses the 'ocd_maxbytes' value to inform the client
+of the maximum OST object size for this target. A file that is striped
+uniformly across multiple OST objects (RAID-0) cannot be larger than the
+number of stripes times the minimum 'ocd_maxbytes' value from any of its
+consituent objects.
+
+The additional space in the 'obd_connect_data' structure is unused and
+reserved for future use.
+
+Other OBD_CONNECT_* flags have no corresponding field in
+obd_connect_data but still control client-server supported features.
+
+If the OBD_CONNECT_RDONLY flag is set then the client is mounted in
+read-only mode and the server honors that by denying any modification
+from this client.
+
+If the OBD_CONNECT_SRVLOCK flag is set then the client and server
+support lockless IO. The server will take locks for client IO requests
+with the OBD_BRW_SRVLOCK flag in the 'niobuf_remote' structure
+flags. This is used for Direct IO or when there is significant lock
+contention on a single OST object. The client takes no LDLM lock and
+delegates locking to the server.
+
+If the OBD_CONNECT_ACL flag is set then the server supports the ACL
+mount option for its filesystem. If the server is mounted with ACL
+support but the client does not pass OBD_CONNECT_ACL then the client
+mount is refused.
+
+If the OBD_CONNECT_XATTR flag is set then the server supports user
+extended attributes. This is requested by the client if mounted
+with the appropriate mount option, but is enabled or disabled by the
+mount options of the backend file system of MDT0000.
+
+If the OBD_CONNECT_TRUNCLOCK flag is set then the client and the
+server support lockless truncate. This is realized in an OST_PUNCH RPC
+by setting the 'obdo' structure's 'o_flag' field to include the
+OBD_FL_SRVLOCK. In that circumstance the client takes no lock, and the
+server must take a lock on the resource while performing the truncate.
+
+If the OBD_CONNECT_ATTRFID flag is set then the server supports
+getattr requests by FID of file instead of name. This reduces
+unnecessary RPCs for DNE and for file attribute revalidation after
+a lock cancellation.
+
+If the OBD_CONNECT_NODEVOH flag is set then the server provides no
+open handle for block and character special inodes.
+
+If the OBD_CONNECT_RMT_CLIENT is set then the client is set as
+'remote' with respect to the server. The client is considered as
+'local' if the user/group database on the client is identical to that
+on the server, otherwise the client is set as 'remote'. This
+terminology is part of Lustre Kerberos feature which is not supported
+as of Lustre 2.7.
+
+If the OBD_CONNECT_RMT_CLIENT_FORCE is set then client is set as
+remote client forcefully. If the server security level doesn't support
+remote clients then this client connect reply will return an -EACCESS
+error.
+
+If the OBD_CONNECT_CANCELSET is set then early batched cancels are
+enabled. The ELC (Early Lock Cancel) feature allows client locks to
+be cancelled prior the cancellation callback if it is clear that lock
+is not needed anymore, for example after rename, after removing file
+or directory, link, etc. This can reduce amount of RPCs significantly.
+
+If the OBD_CONNECT_AT is set then client and server use 'Adaptive
+Timeouts' during request processing. Servers keep track of the RPC
+processing time and report this information back to clients to
+estimate the time needed for future requests and set appropriate RPC
+timeouts.
+
+If the OBD_CONNECT_LRU_RESIZE is set then the LRU self-adjusting is
+enabled.
+
+If the OBD_CONNECT_FID is set then FID support is understood by the
+client and server. This flag was introduced in Lustre 2.0 and
+is required when connecting to any 2.0 or later server. It is
+understood by Lustre 1.8 clients.
+
+If the OBD_CONNECT_VBR is set then version based recovery is used on
+the server. VBR stores the most recent transaction in which each
+object was modified to track its changes on the server and to decide
+if a replayed RPC can be applied during recovery or not. This helps
+to complete recovery even if some clients were missed or evicted.
+That flag is always set by clients since Lustre 1.8 and is used to
+notify the server if client supports VBR.
+
+If the OBD_CONNECT_LOV_V3 is set if the client supports LOV_MAGIC_V3
+(0x0BD30BD0) style layouts. This type of the layout was introduced
+along with OST pools support and added the 'lov_mds_md' layout. The
+OBD_CONNECT_LOV_V3 flag notifies a server if client supports
+this type of LOV EA to handle requests from it properly.
+
+If the OBD_CONNECT_GRANT_SHRINK is set then the client can release
+grant space when idle.
+
+If the OBD_CONNECT_SKIP_ORPHAN is set then OST doesn't reuse orphan
+object IDs after recovery. This connection flag is used between MDS
+and OST to agree about an object pre-creation policy after MDS
+recovery. If set, then if some of precreated objects weren't used
+when an MDT was restarted then an OST must destroy any unused objects
+rather than re-use those objects.
+
+If the OBD_CONNECT_FULL20 is set then the client is Lustre 2.x client.
+Clients that are using old 1.8 format protocol conventions are not
+allowed to connect to 2.x servers. This flag should be set on all
+connections since Lustre 1.8.5.
+
+If the OBD_CONNECT_LAYOUTLOCK is set then the client supports layout
+lock, which allows the server to revoke the layout of a file from
+a client if the layout has changed (e.g. migration between OSTs or
+changes in replication state). The server will not grant a layout
+lock to the old clients that do not support this feature.
+
+If the OBD_CONNECT_64BITHASH is set then the client supports 64-bit
+directory readdir cookie. The server will also use 64-bit hash mode
+while working with ldiskfs.
+
+If the OBD_CONNECT_JOBSTATS is set then the client fills jobid in
+'ptlrpc_body' so server can provide extended RPC statistics per jobid.
+
+If the OBD_CONNECT_UMASK is set then create uses client umask. This is
+default flag for MDS but is not relevant for OSTs.
+
+If the OBD_CONNECT_LVB_TYPE is set then the variable type of lock
+value block (LVB) is supported by a client. This flag was introduced
+to allow the MDS to return data with an IBITS lock, in addition to
+the OST object attributes returned with an EXTENT lock.
+
+If the OBD_CONNECT_LIGHTWEIGHT is set then this connection is the
+'lightweight' one. A lightweight connection has no entry in last_rcvd
+file, so no recovery is possible. A new lightweight
+connection can be set up while the target is in recovery, locks can
+still be acquired through this connection, although they won't be
+replayed. Such type of connection is used by services like quota
+manager, FLDB, etc.
+
+If the OBD_CONNECT_PINGLESS is set then pings can be suppressed. If
+the client and server have this flag during connection and the ptlrpc
+module on server has the option "suppress_pings", then pings will be
+suppressed for this client. There must be an external mechanism to
+notify the targets of client deaths, via the targets "evict_client"
+'procfs' entries. Pings can be disabled on OSTs only.
+
+If the OBD_CONNECT_FLOCK_DEAD is set then the client support flock
+cancellation, which is used for the flock deadlock detection mechanism.
+
+If the OBD_CONNECT_DISP_STRIPE is set then server returns a 'create
+stripe' disposition for open request from the client. This helps to
+optimize a recovery of open requests.
+
+If the OBD_CONNECT_OPEN_BY_FID is set then an open by FID won't pack
+the name in a request. This is used by HSM or other ChangeLog consumers
+for accessing objects by their FID via .lustre/fid/ instead of by name.
+
+If the OBD_CONNECT_MDS_MDS is set then the current connection is an
+intra-MDS one. Such connections are distinguished because they provide
+more functionality specific to MDS-MDS interoperation such as OUT RPCs.
+
+If the OBD_CONNECT_IMP_RECOV is set then the Imperative Recovery is
+supported. Imperative recovery means the clients are notified
+explicitly when and where a target has restarted after failure.
+
+The OBD_CONNECT_REQPORTAL was used to specify that client may use
+OST_REQUEST_PORTAL for requests to don't interfere with IO portal,
+e.g. for MDS-OST interaction. Now it is default request portal for OSC
+and this flag does nothing though it is still set on client side
+during connection process.
+
+The OBD_CONNECT_SOM flag was used to signal that MDS is capable to
+store file size in file attributes, so client may get it directly from
+MDS avoiding glimpse request to OSTs. This feature was implemented as
+demo feature and wasn't enabled by default. Finally it was removed in
+Lustre 2.7 because it causes quite complex recovery cases to handle
+with relatevely small benefits.
+
+The OBD_CONNECT_QUOTA64 was used prior Lustre 2.4 for quota purposes,
+it is obsoleted due to new quota design.
+
+The OBD_CONNECT_REAL is not real connection flag but used locally on
+client to distinguish real connection from local connections between
+layers.
+
+The OBD_CONNECT_CHANGE_QS was used prior Lustre 2.4 for quota, but
+it is obsoleted now due to new quota design.
+
+If the OBD_CONNECT_EINPROGRESS is set then client handles -EINPROGRESS
+RPC error properly. The quota design requires that client must resend
+request with -EINPROGRESS error indefinitely, until successful
+completion or another error. This flag is set on both client and
+server by default. Meanwhile this flag is not checked anywere, so does
+nothing.
+
+If the OBD_CONNECT_FLOCK_OWNER is set then 1.8 clients has fixed flock
+policy and 2.x servers recognize them correctly. Meanwhile this flag
+is not checked anywhere, so does nothing.
+
+If the OBD_CONNECT_NANOSEC_TIME is set then nanosecond timestamps are
+enabled. This flag is not used yet, but reserved for future use.
+
+If the OBD_CONNECT_SHORTIO is set then short IO feature is enabled on
+server. The server will avoid bulk IO for small amount of data but
+data is encapsulated into ptlrpc request/reply. This flag is reserved
+for future use and does nothing yet.
git://git.whamcloud.com / doc / protocol.git / blobdiff