Connections Between Lustre Entities
------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[[connection]]
The Lustre protocol is connection-based in that each two entities
reverse-import uses the same structure as a regular import, and the
reverse-export uses the same structure as a regular export.
-Connection Structures
-~~~~~~~~~~~~~~~~~~~~~
+#################################################################
+Fixme: Move the obd_connect_data.txt include to where it first
+gets introduced. Perhaps in obd_connect_client.txt?
+#################################################################
-Connect Data
-^^^^^^^^^^^^
+include::obd_connect_data.txt[]
-An 'obd_connect_data' structure accompanies every connect operation in
-both the request message and in the reply message.
+include::import.txt[]
-----
-struct obd_connect_data {
- __u64 ocd_connect_flags;
- __u32 ocd_version; /* OBD_CONNECT_VERSION */
- __u32 ocd_grant; /* OBD_CONNECT_GRANT */
- __u32 ocd_index; /* OBD_CONNECT_INDEX */
- __u32 ocd_brw_size; /* OBD_CONNECT_BRW_SIZE */
- __u64 ocd_ibits_known; /* OBD_CONNECT_IBITS */
- __u8 ocd_blocksize; /* OBD_CONNECT_GRANT_PARAM */
- __u8 ocd_inodespace; /* OBD_CONNECT_GRANT_PARAM */
- __u16 ocd_grant_extent; /* OBD_CONNECT_GRANT_PARAM */
- __u32 ocd_unused;
- __u64 ocd_transno; /* OBD_CONNECT_TRANSNO */
- __u32 ocd_group; /* OBD_CONNECT_MDS */
- __u32 ocd_cksum_types; /* OBD_CONNECT_CKSUM */
- __u32 ocd_max_easize; /* OBD_CONNECT_MAX_EASIZE */
- __u32 ocd_instance;
- __u64 ocd_maxbytes; /* OBD_CONNECT_MAXBYTES */
- __u64 padding1;
- __u64 padding2;
- __u64 padding3;
- __u64 padding4;
- __u64 padding5;
- __u64 padding6;
- __u64 padding7;
- __u64 padding8;
- __u64 padding9;
- __u64 paddingA;
- __u64 paddingB;
- __u64 paddingC;
- __u64 paddingD;
- __u64 paddingE;
- __u64 paddingF;
-};
-----
+include::export.txt[]
-The 'ocd_connect_flags' field encodes the connect flags giving the
-capabilities of a connection between client and target. Several of
-those flags (noted in comments above and the discussion below)
-actually control whether the remaining fields of 'obd_connect_data'
-get used. The [[connect-flags]] flags are:
+include::timeouts.txt[]
-----
-#define OBD_CONNECT_RDONLY 0x1ULL /*client has read-only access*/
-#define OBD_CONNECT_INDEX 0x2ULL /*connect specific LOV idx */
-#define OBD_CONNECT_MDS 0x4ULL /*connect from MDT to OST */
-#define OBD_CONNECT_GRANT 0x8ULL /*OSC gets grant at connect */
-#define OBD_CONNECT_SRVLOCK 0x10ULL /*server takes locks for cli */
-#define OBD_CONNECT_VERSION 0x20ULL /*Lustre versions in ocd */
-#define OBD_CONNECT_REQPORTAL 0x40ULL /*Separate non-IO req portal */
-#define OBD_CONNECT_ACL 0x80ULL /*access control lists */
-#define OBD_CONNECT_XATTR 0x100ULL /*client use extended attr */
-#define OBD_CONNECT_CROW 0x200ULL /*MDS+OST create obj on write*/
-#define OBD_CONNECT_TRUNCLOCK 0x400ULL /*locks on server for punch */
-#define OBD_CONNECT_TRANSNO 0x800ULL /*replay sends init transno */
-#define OBD_CONNECT_IBITS 0x1000ULL /*support for inodebits locks*/
-#define OBD_CONNECT_JOIN 0x2000ULL /*files can be concatenated.
- *We do not support JOIN FILE
- *anymore, reserve this flags
- *just for preventing such bit
- *to be reused.*/
-#define OBD_CONNECT_ATTRFID 0x4000ULL /*Server can GetAttr By Fid*/
-#define OBD_CONNECT_NODEVOH 0x8000ULL /*No open hndl on specl nodes*/
-#define OBD_CONNECT_RMT_CLIENT 0x10000ULL /*Remote client */
-#define OBD_CONNECT_RMT_CLIENT_FORCE 0x20000ULL /*Remote client by force */
-#define OBD_CONNECT_BRW_SIZE 0x40000ULL /*Max bytes per rpc */
-#define OBD_CONNECT_QUOTA64 0x80000ULL /*Not used since 2.4 */
-#define OBD_CONNECT_MDS_CAPA 0x100000ULL /*MDS capability */
-#define OBD_CONNECT_OSS_CAPA 0x200000ULL /*OSS capability */
-#define OBD_CONNECT_CANCELSET 0x400000ULL /*Early batched cancels. */
-#define OBD_CONNECT_SOM 0x800000ULL /*Size on MDS */
-#define OBD_CONNECT_AT 0x1000000ULL /*client uses AT */
-#define OBD_CONNECT_LRU_RESIZE 0x2000000ULL /*LRU resize feature. */
-#define OBD_CONNECT_MDS_MDS 0x4000000ULL /*MDS-MDS connection */
-#define OBD_CONNECT_REAL 0x8000000ULL /*real connection */
-#define OBD_CONNECT_CHANGE_QS 0x10000000ULL /*Not used since 2.4 */
-#define OBD_CONNECT_CKSUM 0x20000000ULL /*support several cksum algos*/
-#define OBD_CONNECT_FID 0x40000000ULL /*FID is supported by server */
-#define OBD_CONNECT_VBR 0x80000000ULL /*version based recovery */
-#define OBD_CONNECT_LOV_V3 0x100000000ULL /*client supports LOV v3 EA */
-#define OBD_CONNECT_GRANT_SHRINK 0x200000000ULL /* support grant shrink */
-#define OBD_CONNECT_SKIP_ORPHAN 0x400000000ULL /* don't reuse orphan objids */
-#define OBD_CONNECT_MAX_EASIZE 0x800000000ULL /* preserved for large EA */
-#define OBD_CONNECT_FULL20 0x1000000000ULL /* it is 2.0 client */
-#define OBD_CONNECT_LAYOUTLOCK 0x2000000000ULL /* client uses layout lock */
-#define OBD_CONNECT_64BITHASH 0x4000000000ULL /* client supports 64-bits
- * directory hash */
-#define OBD_CONNECT_MAXBYTES 0x8000000000ULL /* max stripe size */
-#define OBD_CONNECT_IMP_RECOV 0x10000000000ULL /* imp recovery support */
-#define OBD_CONNECT_JOBSTATS 0x20000000000ULL /* jobid in ptlrpc_body */
-#define OBD_CONNECT_UMASK 0x40000000000ULL /* create uses client umask */
-#define OBD_CONNECT_EINPROGRESS 0x80000000000ULL /* client handles -EINPROGRESS
- * RPC error properly */
-#define OBD_CONNECT_GRANT_PARAM 0x100000000000ULL/* extra grant params used for
- * finer space reservation */
-#define OBD_CONNECT_FLOCK_OWNER 0x200000000000ULL /* for the fixed 1.8
- * policy and 2.x server */
-#define OBD_CONNECT_LVB_TYPE 0x400000000000ULL /* variable type of LVB */
-#define OBD_CONNECT_NANOSEC_TIME 0x800000000000ULL /* nanosecond timestamps */
-#define OBD_CONNECT_LIGHTWEIGHT 0x1000000000000ULL/* lightweight connection */
-#define OBD_CONNECT_SHORTIO 0x2000000000000ULL/* short io */
-#define OBD_CONNECT_PINGLESS 0x4000000000000ULL/* pings not required */
-#define OBD_CONNECT_FLOCK_DEAD 0x8000000000000ULL/* deadlock detection */
-#define OBD_CONNECT_DISP_STRIPE 0x10000000000000ULL/* create stripe disposition*/
-#define OBD_CONNECT_OPEN_BY_FID 0x20000000000000ULL /* open by fid won't pack
- name in request */
-----
+include::eviction.txt[]
-Each flag corresponds to a particular capability that the client and
-target together will honor. A client will send a message including
-some subset of these capabilities during a connection request to a
-specific target. It tells the server what capabilities it has. The
-server then replies with the subset of those capabilities it agrees to
-honor (for the given target).
+include::recovery.txt[]
-If the OBD_CONNECT_VERSION flag is set then the 'ocd_version' field is
-honored. The 'ocd_version' gives an encoding of the Lustre
-version. For example, Version 2.7.32 would be hexadecimal number
-0x02073200.
-
-If the OBD_CONNECT_GRANT flag is set then the 'ocd_grant' field is
-honored. The 'ocd_grant' value in a reply (to a connection request)
-sets the client's grant.
-
-If the OBD_CONNECT_INDEX flag is set then the 'ocd_index' field is
-honored. The 'ocd_index' value is set in a reply to a connection
-request. It holds the LOV index of the target.
-
-If the OBD_CONNECT_BRW_SIZE flag is set then the 'ocd_brw_size' field
-is honored. The 'ocd_brw_size' value sets the size of the maximum
-supported RPC. The client proposes a value in its connection request,
-and the server's reply will either agree or further limit the size.
-
-If the OBD_CONNECT_IBITS flag is set then the 'ocd_ibits_known' field
-is honored. The 'ocd_ibits_known' value determines the handling of
-locks on inodes. See the discussion of inodes and extended attributes.
-
-If the OBD_CONNECT_GRANT_PARAM flag is set then the 'ocd_blocksize',
-'ocd_inodespace', and 'ocd_grant_extent' fields are honored. A server
-reply uses the 'ocd_blocksize' value to inform the client of the log
-base two of the size in bytes of the backend file system's blocks.
-
-A server reply uses the 'ocd_inodespace' value to inform the client of
-the log base two of the size of an inode.
-
-Under some circumstances (for example when ZFS is the back end file
-system) there may be additional overhead in handling writes for each
-extent. The server uses the 'ocd_grant_extent' value to inform the
-client of the size in bytes consumed from its grant on the server when
-creating a new file. The client uses this value in calculating how
-much dirty write cache it has and whether it has reached the limit
-established by the target's grant.
-
-If the OBD_CONNECT_TRANSNO flag is set then the 'ocd_transno' field is
-honored. A server uses the 'ocd_transno' value during recovery to
-inform the client of the transaction number at which it should begin
-replay.
-
-If the OBD_CONNECT_MDS flag is set then the 'ocd_group' field is
-honored. When an MDT connects to an OST the 'ocd_group' field informs
-the OSS of the MDT's index. Objects on that OST for that MDT will be
-in a common namespace served by that MDT.
-
-If the OBD_CONNECT_CKSUM flag is set then the 'ocd_cksum_types' field
-is honored. The client uses the 'ocd_checksum_types' field to propose
-to the server the client's available (presumably hardware assisted)
-checksum mechanisms. The server replies with the checksum types it has
-available. Finally, the client will employ the fastest of the agreed
-mechanisms.
-
-If the OBD_CONNECT_MAX_EASIZE flag is set then the 'ocd_max_easize'
-field is honored. The server uses 'ocd_max_easize' to inform the
-client about the amount of space that can be allocated in each inode
-for extended attributes. The 'ocd_max_easize' specifically refers to
-the space used for striping information. This allows the client to
-determine the maximum layout size (and hence stripe count) that can be
-stored on the MDT.
-
-The 'ocd_instance' field (alone) is not governed by an OBD_CONNECT_*
-flag. The MGS uses the 'ocd_instance' value in its reply to a
-connection request to inform the server and target of the "era" of its
-connection. The MGS initializes the era value for each server to zero
-and increments that value every time the target connects. This
-supports imperative recovery.
-
-If the OBD_CONNECT_MAXBYTES flag is set then the 'ocd_maxbytes' field
-is honored. An OSS uses the 'ocd_maxbytes' value to inform the client
-of the maximum OST object size for this target. A stripe on any OST
-for a multi-striped file cannot be larger than the minimum maxbytes
-value.
-
-The additional space in the 'obd_connect_data' structure is unused and
-reserved for future use.
-
-Other OBD_CONNECT_* flags have no corresponding field in
-obd_connect_data but still control client-server supported features.
-
-If the OBD_CONNECT_RDONLY flag is set then the client is mounted in
-read-only mode and the server honors that by denying any modification
-from this client.
-
-If the OBD_CONNECT_SRVLOCK flag is set then the client and server
-support lockless IO. The server will take locks for client IO requests
-with the OBD_BRW_SRVLOCK flag in the 'niobuf_remote' structure
-flags. This is used for Direct IO. The client takes no LDLM lock and
-delegates locking to the server.
-
-If the OBD_CONNECT_ACL flag is set then the server supports the ACL
-mount option for its filesystem. The client supports this mount option
-as well, in that case.
-
-If the OBD_CONNECT_XATTR flag is set then the server supports user
-extended attributes. This is defined by the mount options of the
-servers of the backend file systems and is reflected on the client
-side by the same mount option but for the Lustre file system itself.
-
-If the OBD_CONNECT_TRUNCLOCK flag is set then the client and the
-server support lockless truncate. This is realized in an OST_PUNCH RPC
-by setting the 'obdo' sturcture's 'o_flag' field to include the
-OBD_FL_SRVLOCK. In that circumstance the client takes no lock, and the
-server must take a lock on the resource.
-
-If the OBD_CONNECT_ATTRFID flag is set then the server supports
-getattr requests by FID of file instead of name. This reduces
-unnecessary RPCs for DNE.
-
-If the OBD_CONNECT_NODEVOH flag is set then the server provides no
-open handle for special inodes.
-
-If the OBD_CONNECT_RMT_CLIENT is set then the client is set as
-'remote' with respect to the server. The client is considered as
-'local' if the user/group database on the client is identical to that
-on the server, otherwise the client is set as 'remote'. This
-terminology is part of Lustre Kerberos feature which is not supported
-now.
-
-If the OBD_CONNECT_RMT_CLIENT_FORCE is set then client is set as
-remote client forcefully. If the server security level doesn't support
-remote clients then this client connect reply will return an -EACCESS
-error.
-
-If the OBD_CONNECT_MDS_CAPA is set then MDS supports capability.
-Capabilities are part of Lustre Kerberos. The MDS prepares the
-capability when a file is opened and sends it to a client. A client
-has to present a capability when it wishes to perform an operation on
-that object.
-
-If the OBD_CONNECT_OSS_CAPA is set then OSS supports capability.
-Capabilities are part of Lustre Kerberos. When the clients request the
-OSS to perform a modification operations on objects the capability
-authorizes these operations.
-
-If the OBD_CONNECT_CANCELSET is set then early batched cancels are
-enabled. The ELC (Early Lock Cancel) feature allows client locks to
-be cancelled prior the cancellation callback if it is clear that lock
-is not needed anymore, for example after rename, after removing file
-or directory, link, etc. This can reduce amount of RPCs significantly.
-
-If the OBD_CONNECT_AT is set then client and server use Adaptive
-Timeout while request processing. Servers keep track of the RPCs
-processing time and report this information back to clients to
-estimate the time needed for future requests and set appropriate RPC
-timeouts.
-
-If the OBD_CONNECT_LRU_RESIZE is set then the LRU self-adjusting is
-enabled. This is set by the Lustre configurable option
---enable-lru-resize, and is enabled by default.
-
-If the OBD_CONNECT_FID is set then FID support is required by
-server. This compatibility flag was introduced in Lustre 2.0. All
-servers and clients are using FIDs nowadays. This flag is always set
-on server and used to filter out clients without FID support.
-
-If the OBD_CONNECT_VBR is set then version based recovery is used on
-server. The VBR uses an object version to track its changes on the
-server and to decide if the replay can be applied during recovery
-based on that version. This helps to complete recovery even if some
-clients were missed or evicted. That flag is always set on server
-since Lustre 1.8 and is used just to notify the server if client
-doesn't support VBR.
-
-If the OBD_CONNECT_LOV_V3 is set then the client supports LOV vs
-EA. This type of the LOV extended attribute was introduced along with
-OST pools support and changed the internal structure of that EA. The
-OBD_CONNECT_LOV_V3 flag notifies a server if client doesn't support
-this type of LOV EA to handle requests from it properly.
-
-If the OBD_CONNECT_GRANT_SHRINK is set then the client can release
-grant space when idle.
-
-If the OBD_CONNECT_SKIP_ORPHAN is set then OST doesn't reuse orphan
-object ids after recovery. This connection flag is used between MDS
-and OST to agree about an object pre-creation policy after MDS
-recovery. If some of precreated objects weren't used but an MDT was
-restarted then an OST may re-use not used objects for new pre-create
-request or may not. The latter is preferred and is used by default
-nowadays.
-
-If the OBD_CONNECT_FULL20 is set then the client is Lustre 2.x client.
-Clients that are using old 1.8 format protocol conventions are not
-allowed to connect. This flag should be set on all connections since
-2.0, it is no longer affects behaviour and will be disabled completely
-once Lustre interoperation with old clients is no longer needed.
-
-If the OBD_CONNECT_LAYOUTLOCK is set then the client supports layout
-lock. The server will not grant a layout lock to the old clients
-having no such flag.
-
-If the OBD_CONNECT_64BITHASH is set then the client supports 64-bit
-directory hash. The server will also use 64-bit hash mode while
-working with ldiskfs.
-
-If the OBD_CONNECT_JOBSTATS is set then the client fills jobid in
-'ptlrpc_body' so server can provide extended statistics per jobid.
-
-If the OBD_CONNECT_UMASK is set then create uses client umask. This is
-default flag for MDS but not for OST.
-
-If the OBD_CONNECT_LVB_TYPE is set then the variable type of LVB is
-supported by a client. This flag was introduced along with DNE to
-recognize DNE-aware clients.
-
-If the OBD_CONNECT_LIGHTWEIGHT is set then this connection is the
-'lightweight' one. A lightweight connection has no entry in last_rcvd
-file, so no recovery is possible, at the same time a lightweight
-connection can be set up while the target is in recovery, locks can
-still be acquired through this connection, although they won't be
-replayed. Such type of connection is used by services like quota
-manager, FLDB, etc.
-
-If the OBD_CONNECT_PINGLESS is set then pings can be suppressed. If
-the client and server have this flag during connection and the ptlrpc
-module on server has the option "suppress_pings", then pings will be
-suppressed for this client. There must be an external mechanism to
-notify the targets of client deaths, via the targets "evict_client"
-'procfs' entries. Pings can be disabled on OSTs only.
-
-If the OBD_CONNECT_FLOCK_DEAD is set then the client support flock
-cancellation, which is used for the flock deadlock detection mechanism.
-
-If the OBD_CONNECT_DISP_STRIPE is set then server returns a 'create
-stripe' disposition for open request from the client. This helps to
-optimize a recovery of open requests.
-
-If the OBD_CONNECT_OPEN_BY_FID is set then an open by FID won't pack
-the name in a request. This is used by DNE.
-
-If the OBD_CONNECT_MDS_MDS is set then the current connection is a
-MDS-MDS one. Such connections are distinguished because they provide
-more functionality specific to MDS-MDS interoperation.
-
-If the OBD_CONNECT_IMP_RECOV is set then the Imperative Recovery is
-supported. Imperative recovery means the clients are notified
-explicitly when and where a failed target has restarted.
-
-The OBD_CONNECT_REQPORTAL was used to specify that client may use
-OST_REQUEST_PORTAL for requests to don't interfere with IO portal,
-e.g. for MDS-OST interaction. Now it is default request portal for OSC
-and this flag does nothing though it is still set on client side
-during connection process.
-
-The OBD_CONNECT_CROW flag was used for create-on-write functionality
-on OST, when data objects were created upon first write from the
-client. This wasn't implemented because of complex recovery problems.
-
-The OBD_CONNECT_SOM flag was used to signal that MDS is capable to
-store file size in file attributes, so client may get it directly from
-MDS avoiding glimpse request to OSTs. This feature was implemented as
-demo feature and wasn't enabled by default. Finally it was disabled in
-Lustre 2.7 because it causes quite complex recovery cases to handle
-with relatevely small benefits.
-
-The OBD_CONNECT_JOIN flag was used for the 'join files' feature, which
-allowed files to be concatenated. Lustre no longer supports that
-feature.
-
-The OBD_CONNECT_QUOTA64 was used prior Lustre 2.4 for quota purposes,
-it is obsoleted due to new quota design.
-
-The OBD_CONNECT_REAL is not real connection flag but used locally on
-client to distinguish real connection from local connections between
-layers.
-
-The OBD_CONNECT_CHANGE_QS was used prior Lustre 2.4 for quota needs
-and it is obsoleted now due to new quota design.
-
-If the OBD_CONNECT_EINPROGRESS is set then client handles -EINPROGRESS
-RPC error properly. The quota design requires that client must resend
-request with -EINPROGRESS error indefinitely, until successful
-completion or another error. This flag is set on both client and
-server by default. Meanwhile this flag is not checked anywere, so does
-nothing.
-
-If the OBD_CONNECT_FLOCK_OWNER is set then 1.8 clients has fixed flock
-policy and 2.x servers recognize them correctly. Meanwhile this flag
-is not checked anywhere, so does nothing.
-
-If the OBD_CONNECT_NANOSEC_TIME is set then nanosecond timestamps are
-enabled. This flag is not used nowadays, but reserved for future use.
-
-If the OBD_CONNECT_SHORTIO is set then short IO feature is enabled on
-server. The server will avoid bulk IO for small amount of data but
-data is incapsulated into ptlrpc request/reply. This flag is just
-reserved for future use and does nothing nowadays.
-
-Import
-^^^^^^
-
-----
-#define IMP_STATE_HIST_LEN 16
-struct import_state_hist {
- enum lustre_imp_state ish_state;
- time_t ish_time;
-};
-struct obd_import {
- struct portals_handle imp_handle;
- atomic_t imp_refcount;
- struct lustre_handle imp_dlm_handle;
- struct ptlrpc_connection *imp_connection;
- struct ptlrpc_client *imp_client;
- cfs_list_t imp_pinger_chain;
- cfs_list_t imp_zombie_chain;
- cfs_list_t imp_replay_list;
- cfs_list_t imp_sending_list;
- cfs_list_t imp_delayed_list;
- cfs_list_t imp_committed_list;
- cfs_list_t *imp_replay_cursor;
- struct obd_device *imp_obd;
- struct ptlrpc_sec *imp_sec;
- struct mutex imp_sec_mutex;
- cfs_time_t imp_sec_expire;
- wait_queue_head_t imp_recovery_waitq;
- atomic_t imp_inflight;
- atomic_t imp_unregistering;
- atomic_t imp_replay_inflight;
- atomic_t imp_inval_count;
- atomic_t imp_timeouts;
- enum lustre_imp_state imp_state;
- struct import_state_hist imp_state_hist[IMP_STATE_HIST_LEN];
- int imp_state_hist_idx;
- int imp_generation;
- __u32 imp_conn_cnt;
- int imp_last_generation_checked;
- __u64 imp_last_replay_transno;
- __u64 imp_peer_committed_transno;
- __u64 imp_last_transno_checked;
- struct lustre_handle imp_remote_handle;
- cfs_time_t imp_next_ping;
- __u64 imp_last_success_conn;
- cfs_list_t imp_conn_list;
- struct obd_import_conn *imp_conn_current;
- spinlock_t imp_lock;
- /* flags */
- unsigned long
- imp_no_timeout:1,
- imp_invalid:1,
- imp_deactive:1,
- imp_replayable:1,
- imp_dlm_fake:1,
- imp_server_timeout:1,
- imp_delayed_recovery:1,
- imp_no_lock_replay:1,
- imp_vbr_failed:1,
- imp_force_verify:1,
- imp_force_next_verify:1,
- imp_pingable:1,
- imp_resend_replay:1,
- imp_no_pinger_recover:1,
- imp_need_mne_swab:1,
- imp_force_reconnect:1,
- imp_connect_tried:1;
- __u32 imp_connect_op;
- struct obd_connect_data imp_connect_data;
- __u64 imp_connect_flags_orig;
- int imp_connect_error;
- __u32 imp_msg_magic;
- __u32 imp_msghdr_flags; /* adjusted based on server capability */
- struct ptlrpc_request_pool *imp_rq_pool; /* emergency request pool */
- struct imp_at imp_at; /* adaptive timeout data */
- time_t imp_last_reply_time; /* for health check */
-};
-----
-
-The 'imp_handle' value is the unique id for the import, and is used as
-a hash key to gain access to it. It is not used in any of the Lustre
-protocol messages, but rather is just for internal reference.
-
-The 'imp_refcount' is also for internal use. The value is incremented
-with each RPC created, and decremented as the request is freed. When
-the reference count is zero the import can be freed, as when the
-target is being disconnected.
-
-The 'imp_dlm_handle' is a reference to the LDLM export for this
-client.
-
-There can be multiple paths through the network to a given
-target, in which case there would be multiple 'obd_import_conn' items
-on the 'imp_conn_list'. Each 'obd_imp_conn' includes a
-'ptlrpc_connection', so 'imp_connection' points to the one that is
-actually in use.
-
-The 'imp_client' identifies the (local) portals for sending and
-receiving messages as well as the client's name. The information is
-specific to either an MDC or an OSC.
-
-The 'imp_ping_chain' places the import on a linked list of imports
-that need periodic pings.
-
-The 'imp_zombie_chain' places the import on a list ready for being
-freed. Unused imports ('imp_refcount' is zero) are deleted
-asynchronously by a garbage collecting process.
-
-In order to support recovery the client must keep requests that are in
-the process of being handled by the target. The target replies to a
-request as soon as the target has made its local update to
-memory. When the client receives that reply the request is put on the
-'imp_replay_list'. In the event of a failure (target crash, lost
-message) this list is then replayed for the target during the recovery
-process. When a request has been sent but has not yet received a reply
-it is placed on the 'imp_sending_list'. In the event of a failure
-those will simply be replayed after any recovery has been
-completed. Finally, there may be requests that the client is delaying
-before it sends them. This can happen if the client is in a degraded
-mode, as when it is in recovery after a failure. These requests are
-put on the 'imp_delayed_list' and not processed until recovery is
-complete and the 'imp_sending_list' has been replayed.
-
-In order to support recovery 'open' requests must be preserved even
-after they have completed. Those requests are placed on the
-'imp_committed_list' and the 'imp_replay_cursor' allows for
-accelerated access to those items.
-
-The 'imp_obd' is a reference to the details about the target device
-that is the subject of this import. There is a lot of state info in
-there along with many implementation details that are not relevant to
-the actual Lustre protocol. fixme: I'll want to go through all of the
-fields in that structure to see which, if any need more
-documentation.
-
-The security policy and settings are kept in 'imp_sec', and
-'imp_sec_mutex' helps manage access to that info. The 'imp_sec_expire'
-setting is in support of security policies that have an expiration
-strategy.
-
-Some processes may need the import to be in a fully connected state in
-order to proceed. The 'imp_recovery_waitq' is where those threads will
-wait during recovery.
-
-The 'imp_inflight' field counts the number of in-flight requests. It
-is incremented with each request sent and decremented with each reply
-received.
-
-The client reserves buffers for the processing of requests and
-replies, and then informs LNet about those buffers. Buffers may get
-reused during subsequent processing, but then a point may come when
-the buffer is no longer going to be used. The client increments the
-'imp_unregistering' counter and informs LNet the buffer is no longer
-needed. When LNet has freed the buffer it will notify the client and
-then the 'imp_unregistering' can be decremented again.
-
-During recovery the 'imp_reply_inflight' counts the number of requests
-from the reply list that have been sent and have not been replied to.
-
-The 'imp_inval_count' field counts how many threads are in the process
-of cleaning up this connection or waiting for cleanup to complete. The
-cleanup itself may be needed in the case there is an eviction or other
-problem (fixme what other problem?). The cleanup may involve freeing
-allocated resources, updating internal state, running replay lists,
-and invalidating cache. Since it could take a while there may end up
-multiple threads waiting on this process to complete.
-
-The 'imp_timeout' field is a counter that is incremented every time
-there is a timeout in communication with the target.
-
-The 'imp_state' tracks the state of the import. It draws from the
-enumerated set of values:
-
-.enum_lustre_imp_state
-[options="header"]
-|=====
-| state name | value
-| LUSTRE_IMP_CLOSED | 1
-| LUSTRE_IMP_NEW | 2
-| LUSTRE_IMP_DISCON | 3
-| LUSTRE_IMP_CONNECTING | 4
-| LUSTRE_IMP_REPLAY | 5
-| LUSTRE_IMP_REPLAY_LOCKS | 6
-| LUSTRE_IMP_REPLAY_WAIT | 7
-| LUSTRE_IMP_RECOVER | 8
-| LUSTRE_IMP_FULL | 9
-| LUSTRE_IMP_EVICTED | 10
-|=====
-fixme: what are the transitions between these states? The
-'imp_state_hist' array maintains a list of the last 16
-(IMP_STATE_HIST_LEN) states the import was in, along with the time it
-entered each (fixme: or is it when it left that state?). The list is
-maintained in a circular manner, so the 'imp_state_hist_idx' points to
-the entry in the list for the most recently visited state.
-
-The 'imp_generation' and 'imp_conn_cnt' fields are monotonically
-increasing counters. Every time a connection request is sent to the
-target the 'imp_conn_cnt' counter is incremented, and every time a
-reply is received for the connection request the 'imp_generation'
-counter is incremented.
-
-The 'imp_last_generation_checked' implements an optimization. When a
-replay process has successfully traversed the reply list the
-'imp_generation' value is noted here. If the generation has not
-incremented then the replay list does not need to be traversed again.
-
-During replay the 'imp_last_replay_transno' is set to the transaction
-number of the last request being replayed, and
-'imp_peer_committed_transno is set to the 'pb_last_committed' value
-(of the 'ptlrpc_body') from replies if that value is higher than the
-previous 'imp_peer_committed_transno'. The 'imp_last_transno_checked'
-field implements an optimization. It is set to the
-'imp_last_replay_transno' as its replay is initiated. If
-'imp_last_transno_checked' is still 'imp_last_replay_transno' and
-'imp_generation' is still 'imp_last_generation_checked' then there
-are no additional requests ready to be removed from the replay
-list. Furthermore, 'imp_last_transno_checked' may no longer be needed,
-since the committed transactions are now maintained on a separate list.
-
-The 'imp_remote_handle' is the handle sent by the target in a
-connection reply message to uniquely identify the export for this
-target and client that is maintained on the server. This is the handle
-used in all subsequent messages to the target.
-
-There are two separate ping intervals (fixme: what are the
-values?). If there are no uncommitted messages for the target then the
-default ping interval is used to set the 'imp_next_ping' to the time
-the next ping needs to be sent. If there are uncommitted requests then
-a "short interval" is used to set the time for the next ping.
-
-The 'imp_last_success_conn' value is set to the time of the last
-successful connection. fixme: The source says it is in 64 bit
-jiffies, but does not further indicate how that value is calculated.
-
-Since there can actually be multiple connection paths for a target
-(due to failover or multihomed configurations) the import maintains a
-list of all the possible connection paths in the list pointed to by
-the 'imp_conn_list' field. The 'imp_conn_current' points to the one
-currently in use. Compare with the 'imp_connection' fields. They point
-to different structures, but each is reachable from the other.
-
-Most of the flag, state, and list information in the import needs to
-be accessed atomically. The 'imp_lock' is used to maintain the
-consistency of the import while it is manipulated by multiple threads.
-
-The various flags are documented in the source code and are largely
-obvious from those short comments, reproduced here:
-
-.import flags
-[options="header"]
-|=====
-| flag | explanation
-| imp_no_timeout | timeouts are disabled
-| imp_invalid | client has been evicted
-| imp_deactive | client administratively disabled
-| imp_replayable | try to recover the import
-| imp_dlm_fake | don't run recovery (timeout instead)
-| imp_server_timeout | use 1/2 timeout on MDSs and OSCs
-| imp_delayed_recovery | VBR: imp in delayed recovery
-| imp_no_lock_replay | VBR: if gap was found then no lock replays
-| imp_vbr_failed | recovery by versions was failed
-| imp_force_verify | force an immidiate ping
-| imp_force_next_verify | force a scheduled ping
-| imp_pingable | target is pingable
-| imp_resend_replay | resend for replay
-| imp_no_pinger_recover | disable normal recovery, for test only.
-| imp_need_mne_swab | need IR MNE swab
-| imp_force_reconnect | import must be reconnected, not new connection
-| imp_connect_tried | import has tried to connect with server
-|=====
-A few additional notes are in order. The 'imp_dlm_fake' flag signifies
-that this is not a "real" import, but rather it is a "reverse"import
-in support of the LDLM. When the LDLM invokes callback operations the
-messages are initiated at the other end, so there need to a fake
-import to receive the replies from the operation. Prior to the
-introduction of adaptive timeouts the servers were given fixed timeout
-value that were half those used for the clients. The
-'imp_server_timeout' flag indicated that the import should use the
-half-sized timeouts, but with the introduction of adaptive timeouts
-this facility is no longer used. "VBR" is "version based recovery",
-and it introduces a new possibility for handling requests. Previously,
-f there were a gap in the transaction number sequence the the requests
-associated with the missing transaction numbers would be
-discarded. With VBR those transaction only need to be discarded if
-there is an actual dependency between the ones that were skipped and
-the currently latest committed transaction number. fixme: What are the
-circumstances that would lead to setting the 'imp_force_next_verify'
-or 'imp_pingable' flags? During recovery, the client sets the
-'imp_no_pinger_recover' flag, which tells the process to proceed from
-the current value of 'imp_replay_last_transno'. The
-'imp_need_mne_swab' flag indicates a version dependent circumstance
-where swabbing was inadvertently left out of one processing step.
-
-
-Export
-^^^^^^
-
-An 'obd_export' structure for a given target is created on a server
-for each client that connects to that target. The exports for all the
-clients for a given target are managed together. The export represents
-the connection state between the client and target as well as the
-current state of any ongoing activity. Thus each pending request will
-have a reference to the export. The export is discarded if the
-connection goes away, but only after all the references to it have
-been cleaned up. The state information for each export is also
-maintained on disk. In the event of a server failure, that or another
-server can read the export date from disk to enable recovery.
-
-----
-struct obd_export {
- struct portals_handle exp_handle;
- atomic_t exp_refcount;
- atomic_t exp_rpc_count;
- atomic_t exp_cb_count;
- atomic_t exp_replay_count;
- atomic_t exp_locks_count;
-#if LUSTRE_TRACKS_LOCK_EXP_REFS
- cfs_list_t exp_locks_list;
- spinlock_t exp_locks_list_guard;
-#endif
- struct obd_uuid exp_client_uuid;
- cfs_list_t exp_obd_chain;
- cfs_hlist_node_t exp_uuid_hash;
- cfs_hlist_node_t exp_nid_hash;
- cfs_list_t exp_obd_chain_timed;
- struct obd_device *exp_obd;
- struct obd_import *exp_imp_reverse;
- struct nid_stat *exp_nid_stats;
- struct ptlrpc_connection *exp_connection;
- __u32 exp_conn_cnt;
- cfs_hash_t *exp_lock_hash;
- cfs_hash_t *exp_flock_hash;
- cfs_list_t exp_outstanding_replies;
- cfs_list_t exp_uncommitted_replies;
- spinlock_t exp_uncommitted_replies_lock;
- __u64 exp_last_committed;
- cfs_time_t exp_last_request_time;
- cfs_list_t exp_req_replay_queue;
- spinlock_t exp_lock;
- struct obd_connect_data exp_connect_data;
- enum obd_option exp_flags;
- unsigned long
- exp_failed:1,
- exp_in_recovery:1,
- exp_disconnected:1,
- exp_connecting:1,
- exp_delayed:1,
- exp_vbr_failed:1,
- exp_req_replay_needed:1,
- exp_lock_replay_needed:1,
- exp_need_sync:1,
- exp_flvr_changed:1,
- exp_flvr_adapt:1,
- exp_libclient:1,
- exp_need_mne_swab:1;
- enum lustre_sec_part exp_sp_peer;
- struct sptlrpc_flavor exp_flvr;
- struct sptlrpc_flavor exp_flvr_old[2];
- cfs_time_t exp_flvr_expire[2];
- spinlock_t exp_rpc_lock;
- cfs_list_t exp_hp_rpcs;
- cfs_list_t exp_reg_rpcs;
- cfs_list_t exp_bl_list;
- spinlock_t exp_bl_list_lock;
- union {
- struct tg_export_data eu_target_data;
- struct mdt_export_data eu_mdt_data;
- struct filter_export_data eu_filter_data;
- struct ec_export_data eu_ec_data;
- struct mgs_export_data eu_mgs_data;
- } u;
- struct nodemap *exp_nodemap;
-};
-----
-
-The 'exp_handle' is a little extra information as compared with a
-'struct lustre_handle', which is just the cookie. The cookie that the
-server generates to uniquely identify this connection gets put into
-this structure along with their information about the device in
-question. This is the cookie the *_CONNECT reply sends back to the
-client and is then stored int he client's import.
-
-The 'exp_refcount' gets incremented whenever some aspect of the export
-is "in use". The arrival of an otherwise unprocessed message for this
-target will increment the refcount. A reference by an LDLM lock that
-gets taken will increment the refcount. Callback invocations and
-replay also lead to incrementing the 'ref_count'. The next four fields
-- 'exp_rpc_count', exp_cb_count', and 'exp_replay_count', and
-'exp_locks_count' - all subcategorize the 'exp_refcount'. The
-reference counter keeps the export alive while there are any users of
-that export. The reference counter is also used for debug
-purposes. Similarly, the 'exp_locks_list' and 'exp_locks_list_guard'
-are further debug info that list the actual locks accounted for in
-'exp_locks_count'.
-
-The 'exp_client_uuid' gives the UUID of the client connected to this
-export. Fixme: when and how does the UUID get generated?
-
-The server maintains all the exports for a given target on a circular
-list. Each export's place on that list is maintained in the
-'exp_obd_chain'. A common activity is to look up the export based on
-the UUID or the nid of the client, and the 'exp_uuid_hash' and
-'exp_nid_hash' fields maintain this export's place in hashes
-constructed for that purpose.
-
-Exports are also maintained on a list sorted by the last time the
-corresponding client was heard from. The 'exp_obd_chain_timed' field
-maintains the export's place on that list. When a message arrives from
-the client the time is "now" so the export gets put at the end of the
-list. Since it is circular, the next export is then the oldest. If it
-has not been heard of within its timeout interval that export is
-marked for later eviction.
-
-The 'exp_obd' points to the 'obd_device' structure for the device that
-is the target of this export.
-
-In the event of an LDLM call-back the export needs to have a the ability to
-initiate messages back to the client. The 'exp_imp_reverse' provides a
-"reverse" import that manages this capability.
-
-The '/proc' stats for the export (and the target) get updated via the
-'exp_nid_stats'.
-
-The 'exp_connection' points to the connection information for this
-export. This is the information about the actual networking pathway(s)
-that get used for communication.
-
-
-The 'exp_conn_cnt' notes the connection count value from the client at
-the time of the connection. In the event that more than one connection
-request is issued before the connection is established then the
-'exp_conn_cnt' will list the highest value. If a previous connection
-attempt (with a lower value) arrives later it may be safely
-discarded. Every request lists its connection count, so non-connection
-requests with lower connection count values can also be discarded.
-Note that this does not count how many times the client has connected
-to the target. If a client is evicted the export is deleted once it
-has been cleaned up and its 'exp_ref_count' reduced to zero. A new
-connection from the client will get a new export.
-
-The 'exp_lock_hash' provides access to the locks granted to the
-corresponding client for this target. If a lock cannot be granted it
-is discarded. A file system lock ("flock") is also implemented through
-the LDLM lock system, but not all LDLM locks are flocks. The ones that
-are flocks are gathered in a hash 'exp_flock_hash'. This supports
-deadlock detection.
-
-For those requests that initiate file system modifying transactions
-the request and its attendant locks need to be preserved until either
-a) the client acknowleges recieving the reply, or b) the transaction
-has been committed locally. This ensures a request can be replayed in
-the event of a failure. The LDLM lock is being kept until one of these
-event occurs to prevent any other modifications of the same object.
-The reply is kept on the 'exp_outstanding_replies' list until the LNet
-layer notifies the server that the reply has been acknowledged. A reply
-is kept on the 'exp_uncommitted_replies' list until the transaction
-(if any) has been committed.
-
-The 'exp_last_committed' value keeps the transaction number of the
-last committed transaction. Every reply to a client includes this
-value as a means of early-as-possible notification of transactions that
-have been committed.
-
-The 'exp_last_request_time' is self explanatory.
-
-During reply a request that is waiting for reply is maintained on the
-list 'exp_req_replay_queue'.
-
-The 'exp_lock' spin-lock is used for access control to the exports
-flags, as well as the 'exp_outstanding_replies' list and the revers
-import, if any.
-
-The 'exp_connect_data' refers to an 'obd_connect_data' structure for
-the connection established between this target and the client this
-export refers to. See also the corresponding entry in the import and
-in the connect messages passed between the hosts.
-
-The 'exp_flags' field encodes three directives as follows:
-----
-enum obd_option {
- OBD_OPT_FORCE = 0x0001,
- OBD_OPT_FAILOVER = 0x0002,
- OBD_OPT_ABORT_RECOV = 0x0004,
-};
-----
-fixme: Are the set for some exports and a condition of their
-existence? or do they reflect a transient state the export is passing
-through?
-
-The 'exp_failed' flag gets set whenever the target has failed for any
-reason or the export is otherwise due to be cleaned up. Once set it
-will not be unset in this export. Any subsequent connection between
-the client and the target would be governed by a new export.
-
-After a failure export data is retrieved from disk and the exports
-recreated. Exports created in this way will have their
-'exp_in_recovery' flag set. Once any outstanding requests and locks
-have been recovered for the client, then the export is recovered and
-'exp_in_recovery' can be cleared. When all the client exports for a
-given target have been recovered then the target is considered
-recovered, and when all targets have been recovered the server is
-considered recovered.
-
-A *_DISCONNECT message from the client will set the 'exp_disconnected'
-flag, as will any sort of failure of the target. Once set the export
-will be cleaned up and deleted.
-
-When a *_CONNECT message arrives the 'exp_connecting' flag is set. If
-for some reason a second *_CONNECT request arrives from the client it can
-be discarded when this flag is set.
-
-The 'exp_delayed' flag is no longer used. In older code it indicated
-that recovery had not completed in a timely fashion, but that a tardy
-recovery would still be possible, since there were no dependencies on
-the export.
-
-The 'exp_vbr_failed' flag indicates a failure during the recovery
-process. See <<recovery>> for a more detailed discussion of recovery
-and transaction replay. For a file system modifying request, the
-server composes its reply including the 'pb_pre_versions' entries in
-'ptlrpc_body', which indicate the most recent updates to the
-object. The client updates the request with the 'pb_transno' and
-'pb_pre_versions' from the reply, and keeps that request until the
-target signals that the transaction has been committed to disk. If the
-client times-out without that confirmation then it will 'replay' the
-request, which now includes the 'pb_pre_versions' information. During
-a replay the target checks that the object has the same version as
-'pb_pre_versions' in replay. If this check fails then the object can't
-be restored in the same state as it was in before failure. Usually that
-happens if the recovery process fails for the connection between some
-other client and this target, so part of change needed for this client
-wasn't restored. At that point the 'exp_vbr_failed' flag is set
-to indicate version based recovery failed. This will lead to the client
-being evicted and this export being cleaned up and deleted.
-
-At the start of recovery both the 'exp_req_replay_needed' and
-'exp_lock_replay_needed' flags are set. As request replay is completed
-the 'exp_req_replay_needed' flag is cleared. As lock replay is
-completed the 'exp_lock_replay_needed' flag is cleared. Once both are
-cleared the 'exp_in_recovery' flag can be cleared.
-
-The 'exp_need_sync' supports an optimization. At mount time it is
-likely that every client (potentially thousands) will create an export
-and that export will need to be saved to disk synchronously. This can
-lead to an unusually high and poorly performing interaction with the
-disk. When the export is created the 'exp_need_sync' flag is set and
-the actual writing to disk is delayed. As transactions arrive from
-clients (in a much less coordinated fashion) the 'exp_need_sync' flag
-indicates a need to have the export data on disk before proceeding
-with a new transaction, so as it is next updated the transaction is
-done synchronously to commit all changes on disk. At that point the
-flag is cleared (except see below).
-
-In DNE (phase I) the export for an MDT managing the connection from
-another MDT will want to always keep the 'exp_need_sync' flag set. For
-that special case such an export sets the 'exp_keep_sync', which then
-prevents the 'exp_need_sync' flag from ever being cleared. This will
-no longer be needed in DNE Phase II.
-
-The 'exp_flvr_changed' and 'exp_flvr_adapt' flags along with
-'exp_sp_peer', 'exp_flvr', 'exp_flvr_old', and 'exp_flvr_expire'
-fields are all used to manage the security settings for the
-connection. Security is discussed in the <<security>> section. (fixme:
-or will be.)
-
-The 'exp_libclient' flag indicates that the export is for a client
-based on "liblustre". This allows for simplified handling on the
-server. (fixme: how is processing simplified? It sounds like I may
-need a whole special section on liblustre.)
-
-The 'exp_need_mne_swab' flag indicates the presence of an old bug that
-affected one special case of failed swabbing. It is not part of
-current processing.
-
-As RPCs arrive they are first subjected to triage. Each request is
-placed on the 'exp_hp_rpcs' list and examined to see if it is high
-priority (PING, truncate, bulk I/O). If it is not high priority then
-it is moved to the 'exp_reg_prcs' list. The 'exp_rpc_lock' protects
-both lists from concurrent access.
-
-All arriving LDLM requests get put on the 'exp_bl_list' and access to
-that list is controlled via the 'exp_bl_list_lock'.
-
-The union provides for target specific data. The 'eu_target_data' is
-for a common core of fields for a generic target. The others are
-specific to particular target types: 'eu_mdt_data' for MDTs,
-'eu_filter_data' for OSTs, 'eu_ec_data' for an "echo client" (fixme:
-describe what an echo client is somewhere), and 'eu_mgs_data' is for
-an MGS.
-
-The 'exp_bl_lock_at' field supports adaptive timeouts which will be
-discussed separately. (fixme: so discuss it somewhere.)
-
-Connection Count
-^^^^^^^^^^^^^^^^
-
-Each export maintains a connection count.
git://git.whamcloud.com / doc / protocol.git / blobdiff