Doxygen comments for LNet API. This is a comment-only change.
i=maxim
FILE_PATTERNS = *.h \
*.c
RECURSIVE = YES
-EXCLUDE =
+EXCLUDE = lnet/klnds/ciblnd/ \
+ lnet/klnds/gmlnd/ \
+ lnet/klnds/iiblnd/ \
+ lnet/klnds/mxlnd/ \
+ lnet/klnds/openiblnd/ \
+ lnet/klnds/qswlnd/ \
+ lnet/klnds/ralnd/ \
+ lnet/klnds/viblnd/
EXCLUDE_SYMLINKS = NO
-EXCLUDE_PATTERNS =
+EXCLUDE_PATTERNS = */darwin/*
EXCLUDE_SYMBOLS = EXPORT_SYMBOL*
EXAMPLE_PATH =
EXAMPLE_PATTERNS = *
FILE_PATTERNS = *.h \
*.c
RECURSIVE = YES
-EXCLUDE =
+EXCLUDE = lnet/klnds/ciblnd/ \
+ lnet/klnds/gmlnd/ \
+ lnet/klnds/iiblnd/ \
+ lnet/klnds/mxlnd/ \
+ lnet/klnds/openiblnd/ \
+ lnet/klnds/qswlnd/ \
+ lnet/klnds/ralnd/ \
+ lnet/klnds/viblnd/
EXCLUDE_SYMLINKS = NO
-EXCLUDE_PATTERNS =
+EXCLUDE_PATTERNS = */darwin/*
EXCLUDE_SYMBOLS = EXPORT_SYMBOL*
EXAMPLE_PATH =
EXAMPLE_PATTERNS = *
int cfs_parse_nidlist(char *str, int len, cfs_list_t *list);
int cfs_match_nid(lnet_nid_t nid, cfs_list_t *list);
+/** \addtogroup lnet_addr
+ * @{ */
/* how an LNET NID encodes net:address */
+/** extract the address part of an lnet_nid_t */
#define LNET_NIDADDR(nid) ((__u32)((nid) & 0xffffffff))
+/** extract the network part of an lnet_nid_t */
#define LNET_NIDNET(nid) ((__u32)(((nid) >> 32)) & 0xffffffff)
+/** make an lnet_nid_t from a network part and an address part */
#define LNET_MKNID(net,addr) ((((__u64)(net))<<32)|((__u64)(addr)))
/* how net encodes type:number */
#define LNET_NETNUM(net) ((net) & 0xffff)
#define LNET_NETTYP(net) (((net) >> 16) & 0xffff)
#define LNET_MKNET(typ,num) ((((__u32)(typ))<<16)|((__u32)(num)))
+/** @} lnet_addr */
/* max value for numeric network address */
#define MAX_NUMERIC_VALUE 0xffffffff
/*
* Number of outstanding watchdogs.
* When it hits 1, we start the dispatcher.
- * When it hits 0, we stop the distpatcher.
+ * When it hits 0, we stop the dispatcher.
*/
static __u32 lcw_refcount = 0;
static CFS_DECLARE_MUTEX(lcw_refcount_sem);
#ifndef __LNET_API_H__
#define __LNET_API_H__
+/** \defgroup lnet LNet
+ *
+ * The Lustre Networking subsystem.
+ *
+ * LNet is an asynchronous message-passing API, which provides an unreliable
+ * connectionless service that can't guarantee any order. It supports OFA IB,
+ * TCP/IP, and Cray Portals, and routes between heterogeneous networks.
+ *
+ * LNet can run both in OS kernel space and in userspace as a library.
+ * @{
+ */
+
#include <lnet/types.h>
+/** \defgroup lnet_init_fini Initialization and cleanup
+ * The LNet must be properly initialized before any LNet calls can be made.
+ * @{ */
int LNetInit(void);
void LNetFini(void);
int LNetNIInit(lnet_pid_t requested_pid);
int LNetNIFini(void);
+/** @} lnet_init_fini */
+/** \defgroup lnet_addr LNet addressing and basic types
+ *
+ * Addressing scheme and basic data types of LNet.
+ *
+ * The LNet API is memory-oriented, so LNet must be able to address not only
+ * end-points but also memory region within a process address space.
+ * An ::lnet_nid_t addresses an end-point. An ::lnet_pid_t identifies a process
+ * in a node. A portal represents an opening in the address space of a
+ * process. Match bits is criteria to identify a region of memory inside a
+ * portal, and offset specifies an offset within the memory region.
+ *
+ * LNet creates a table of portals for each process during initialization.
+ * This table has MAX_PORTALS entries and its size can't be dynamically
+ * changed. A portal stays empty until the owning process starts to add
+ * memory regions to it. A portal is sometimes called an index because
+ * it's an entry in the portals table of a process.
+ *
+ * \see LNetMEAttach
+ * @{ */
int LNetGetId(unsigned int index, lnet_process_id_t *id);
int LNetDist(lnet_nid_t nid, lnet_nid_t *srcnid, __u32 *order);
-int LNetCtl(unsigned int cmd, void *arg);
-void LNetSnprintHandle (char *str, int str_len, lnet_handle_any_t handle);
+void LNetSnprintHandle(char *str, int str_len, lnet_handle_any_t handle);
-/*
- * Portals
- */
-int LNetSetLazyPortal(int portal);
-int LNetClearLazyPortal(int portal);
+/** @} lnet_addr */
-/*
- * Match entries
- */
+
+/** \defgroup lnet_me Match entries
+ *
+ * A match entry (abbreviated as ME) describes a set of criteria to accept
+ * incoming requests.
+ *
+ * A portal is essentially a match list plus a set of attributes. A match
+ * list is a chain of MEs. Each ME includes a pointer to a memory descriptor
+ * and a set of match criteria. The match criteria can be used to reject
+ * incoming requests based on process ID or the match bits provided in the
+ * request. MEs can be dynamically inserted into a match list by LNetMEAttach()
+ * and LNetMEInsert(), and removed from its list by LNetMEUnlink().
+ * @{ */
int LNetMEAttach(unsigned int portal,
lnet_process_id_t match_id_in,
__u64 match_bits_in,
lnet_handle_me_t *handle_out);
int LNetMEUnlink(lnet_handle_me_t current_in);
+/** @} lnet_me */
-/*
- * Memory descriptors
- */
+/** \defgroup lnet_md Memory descriptors
+ *
+ * A memory descriptor contains information about a region of a user's
+ * memory (either in kernel or user space) and optionally points to an
+ * event queue where information about the operations performed on the
+ * memory descriptor are recorded. Memory descriptor is abbreviated as
+ * MD and can be used interchangeably with the memory region it describes.
+ *
+ * The LNet API provides two operations to create MDs: LNetMDAttach()
+ * and LNetMDBind(); one operation to unlink and release the resources
+ * associated with a MD: LNetMDUnlink().
+ * @{ */
int LNetMDAttach(lnet_handle_me_t current_in,
lnet_md_t md_in,
lnet_unlink_t unlink_in,
lnet_handle_md_t *handle_out);
int LNetMDUnlink(lnet_handle_md_t md_in);
+/** @} lnet_md */
-/*
- * Event queues
- */
+/** \defgroup lnet_eq Events and event queues
+ *
+ * Event queues (abbreviated as EQ) are used to log operations performed on
+ * local MDs. In particular, they signal the completion of a data transmission
+ * into or out of a MD. They can also be used to hold acknowledgments for
+ * completed PUT operations and indicate when a MD has been unlinked. Multiple
+ * MDs can share a single EQ. An EQ may have an optional event handler
+ * associated with it. If an event handler exists, it will be run for each
+ * event that is deposited into the EQ.
+ *
+ * In addition to the lnet_handle_eq_t, the LNet API defines two types
+ * associated with events: The ::lnet_event_kind_t defines the kinds of events
+ * that can be stored in an EQ. The lnet_event_t defines a structure that
+ * holds the information about with an event.
+ *
+ * There are five functions for dealing with EQs: LNetEQAlloc() is used to
+ * create an EQ and allocate the resources needed, while LNetEQFree()
+ * releases these resources and free the EQ. LNetEQGet() retrieves the next
+ * event from an EQ, and LNetEQWait() can be used to block a process until
+ * an EQ has at least one event. LNetEQPoll() can be used to test or wait
+ * on multiple EQs.
+ * @{ */
int LNetEQAlloc(unsigned int count_in,
lnet_eq_handler_t handler,
lnet_handle_eq_t *handle_out);
int timeout_ms,
lnet_event_t *event_out,
int *which_eq_out);
+/** @} lnet_eq */
-/*
- * Data movement
- */
+/** \defgroup lnet_data Data movement operations
+ *
+ * The LNet API provides two data movement operations: LNetPut()
+ * and LNetGet().
+ * @{ */
int LNetPut(lnet_nid_t self,
lnet_handle_md_t md_in,
lnet_ack_req_t ack_req_in,
unsigned int portal_in,
__u64 match_bits_in,
unsigned int offset_in);
+/** @} lnet_data */
+
+/** \defgroup lnet_misc Miscellaneous operations.
+ * Miscellaneous operations.
+ * @{ */
+int LNetSetLazyPortal(int portal);
+int LNetClearLazyPortal(int portal);
+int LNetCtl(unsigned int cmd, void *arg);
int LNetSetAsync(lnet_process_id_t id, int nasync);
#ifndef __KERNEL__
* lnet/lib-types.h */
void lnet_server_mode();
-#endif
+#endif
+/** @} lnet_misc */
+/** @} lnet */
#endif
#ifndef __LNET_TYPES_H__
#define __LNET_TYPES_H__
+/** \addtogroup lnet
+ * @{ */
+
#include <libcfs/libcfs.h>
-#define LNET_RESERVED_PORTAL 0 /* portals reserved for lnet's own use */
+/** \addtogroup lnet_addr
+ * @{ */
+
+/** Portal reserved for LNet's own use.
+ * \see lustre/include/lustre/lustre_idl.h for Lustre portal assignments.
+ */
+#define LNET_RESERVED_PORTAL 0
+/**
+ * Address of an end-point in an LNet network.
+ *
+ * A node can have multiple end-points and hence multiple addresses.
+ * An LNet network can be a simple network (e.g. tcp0) or a network of
+ * LNet networks connected by LNet routers. Therefore an end-point address
+ * has two parts: network ID, and address within a network.
+ *
+ * \see LNET_NIDNET, LNET_NIDADDR, and LNET_MKNID.
+ */
typedef __u64 lnet_nid_t;
+/**
+ * ID of a process in a node. Shortened as PID to distinguish from
+ * lnet_process_id_t, the global process ID.
+ */
typedef __u32 lnet_pid_t;
+/** wildcard NID that matches any end-point address */
#define LNET_NID_ANY ((lnet_nid_t) -1)
+/** wildcard PID that matches any lnet_pid_t */
#define LNET_PID_ANY ((lnet_pid_t) -1)
#ifdef CRAY_XT3
#define LNET_TIME_FOREVER (-1)
+/**
+ * Objects maintained by the LNet are accessed through handles. Handle types
+ * have names of the form lnet_handle_xx_t, where xx is one of the two letter
+ * object type codes ('eq' for event queue, 'md' for memory descriptor, and
+ * 'me' for match entry).
+ * Each type of object is given a unique handle type to enhance type checking.
+ * The type lnet_handle_any_t can be used when a generic handle is needed.
+ * Every handle value can be converted into a value of type lnet_handle_any_t
+ * without loss of information.
+ */
typedef struct {
__u64 cookie;
} lnet_handle_any_t;
#define LNET_WIRE_HANDLE_COOKIE_NONE (-1)
+/**
+ * Invalidate handle \a h.
+ */
static inline void LNetInvalidateHandle(lnet_handle_any_t *h)
{
h->cookie = LNET_WIRE_HANDLE_COOKIE_NONE;
}
+/**
+ * Compare handles \a h1 and \a h2.
+ *
+ * \return 1 if handles are equal, 0 if otherwise.
+ */
static inline int LNetHandleIsEqual (lnet_handle_any_t h1, lnet_handle_any_t h2)
{
return (h1.cookie == h2.cookie);
}
+/**
+ * Check whether handle \a h is invalid.
+ *
+ * \return 1 if handle is invalid, 0 if valid.
+ */
static inline int LNetHandleIsInvalid(lnet_handle_any_t h)
{
return (LNET_WIRE_HANDLE_COOKIE_NONE == h.cookie);
}
+/**
+ * Global process ID.
+ */
typedef struct {
+ /** node id */
lnet_nid_t nid;
- lnet_pid_t pid; /* node id / process id */
+ /** process id */
+ lnet_pid_t pid;
} lnet_process_id_t;
+/** @} lnet_addr */
+
+/** \addtogroup lnet_me
+ * @{ */
+/**
+ * Specifies whether the match entry or memory descriptor should be unlinked
+ * automatically (LNET_UNLINK) or not (LNET_RETAIN).
+ */
typedef enum {
LNET_RETAIN = 0,
LNET_UNLINK
} lnet_unlink_t;
+/**
+ * Values of the type lnet_ins_pos_t are used to control where a new match
+ * entry is inserted. The value LNET_INS_BEFORE is used to insert the new
+ * entry before the current entry or before the head of the list. The value
+ * LNET_INS_AFTER is used to insert the new entry after the current entry
+ * or after the last item in the list.
+ */
typedef enum {
LNET_INS_BEFORE,
LNET_INS_AFTER
} lnet_ins_pos_t;
+/** @} lnet_me */
+
+/** \addtogroup lnet_md
+ * @{ */
+
+/**
+ * Defines the visible parts of a memory descriptor. Values of this type
+ * are used to initialize memory descriptors.
+ */
typedef struct {
+ /**
+ * Specify the memory region associated with the memory descriptor.
+ * If the options field has:
+ * - LNET_MD_KIOV bit set: The start field points to the starting
+ * address of an array of lnet_kiov_t and the length field specifies
+ * the number of entries in the array. The length can't be bigger
+ * than LNET_MAX_IOV. The lnet_kiov_t is used to describe page-based
+ * fragments that are not necessarily mapped in virtal memory.
+ * - LNET_MD_IOVEC bit set: The start field points to the starting
+ * address of an array of struct iovec and the length field specifies
+ * the number of entries in the array. The length can't be bigger
+ * than LNET_MAX_IOV. The struct iovec is used to describe fragments
+ * that have virtual addresses.
+ * - Otherwise: The memory region is contiguous. The start field
+ * specifies the starting address for the memory region and the
+ * length field specifies its length.
+ *
+ * When the memory region is fragmented, all fragments but the first
+ * one must start on page boundary, and all but the last must end on
+ * page boundary.
+ */
void *start;
unsigned int length;
+ /**
+ * Specifies the maximum number of operations that can be performed
+ * on the memory descriptor. An operation is any action that could
+ * possibly generate an event. In the usual case, the threshold value
+ * is decremented for each operation on the MD. When the threshold
+ * drops to zero, the MD becomes inactive and does not respond to
+ * operations. A threshold value of LNET_MD_THRESH_INF indicates that
+ * there is no bound on the number of operations that may be applied
+ * to a MD.
+ */
int threshold;
+ /**
+ * Specifies the largest incoming request that the memory descriptor
+ * should respond to. When the unused portion of a MD (length -
+ * local offset) falls below this value, the MD becomes inactive and
+ * does not respond to further operations. This value is only used
+ * if the LNET_MD_MAX_SIZE option is set.
+ */
int max_size;
+ /**
+ * Specifies the behavior of the memory descriptor. A bitwise OR
+ * of the following values can be used:
+ * - LNET_MD_OP_PUT: The LNet PUT operation is allowed on this MD.
+ * - LNET_MD_OP_GET: The LNet GET operation is allowed on this MD.
+ * - LNET_MD_MANAGE_REMOTE: The offset used in accessing the memory
+ * region is provided by the incoming request. By default, the
+ * offset is maintained locally. When maintained locally, the
+ * offset is incremented by the length of the request so that
+ * the next operation (PUT or GET) will access the next part of
+ * the memory region. Note that only one offset variable exists
+ * per memory descriptor. If both PUT and GET operations are
+ * performed on a memory descriptor, the offset is updated each time.
+ * - LNET_MD_TRUNCATE: The length provided in the incoming request can
+ * be reduced to match the memory available in the region (determined
+ * by subtracting the offset from the length of the memory region).
+ * By default, if the length in the incoming operation is greater
+ * than the amount of memory available, the operation is rejected.
+ * - LNET_MD_ACK_DISABLE: An acknowledgment should not be sent for
+ * incoming PUT operations, even if requested. By default,
+ * acknowledgments are sent for PUT operations that request an
+ * acknowledgment. Acknowledgments are never sent for GET operations.
+ * The data sent in the REPLY serves as an implicit acknowledgment.
+ * - LNET_MD_KIOV: The start and length fields specify an array of
+ * lnet_kiov_t.
+ * - LNET_MD_IOVEC: The start and length fields specify an array of
+ * struct iovec.
+ * - LNET_MD_MAX_SIZE: The max_size field is valid.
+ *
+ * Note:
+ * - LNET_MD_KIOV or LNET_MD_IOVEC allows for a scatter/gather
+ * capability for memory descriptors. They can't be both set.
+ * - When LNET_MD_MAX_SIZE is set, the total length of the memory
+ * region (i.e. sum of all fragment lengths) must not be less than
+ * \a max_size.
+ */
unsigned int options;
+ /**
+ * A user-specified value that is associated with the memory
+ * descriptor. The value does not need to be a pointer, but must fit
+ * in the space used by a pointer. This value is recorded in events
+ * associated with operations on this MD.
+ */
void *user_ptr;
+ /**
+ * A handle for the event queue used to log the operations performed on
+ * the memory region. If this argument is a NULL handle (i.e. nullified
+ * by LNetInvalidateHandle()), operations performed on this memory
+ * descriptor are not logged.
+ */
lnet_handle_eq_t eq_handle;
} lnet_md_t;
#define LNET_MTU_BITS 20
#define LNET_MTU (1<<LNET_MTU_BITS)
-/* limit on the number of entries in discontiguous MDs */
+/** limit on the number of fragments in discontiguous MDs */
#define LNET_MAX_IOV 256
/* Max payload size */
# endif
#endif
-/* Options for the MD structure */
+/**
+ * Options for the MD structure. See lnet_md_t::options.
+ */
#define LNET_MD_OP_PUT (1 << 0)
+/** See lnet_md_t::options. */
#define LNET_MD_OP_GET (1 << 1)
+/** See lnet_md_t::options. */
#define LNET_MD_MANAGE_REMOTE (1 << 2)
/* unused (1 << 3) */
+/** See lnet_md_t::options. */
#define LNET_MD_TRUNCATE (1 << 4)
+/** See lnet_md_t::options. */
#define LNET_MD_ACK_DISABLE (1 << 5)
+/** See lnet_md_t::options. */
#define LNET_MD_IOVEC (1 << 6)
+/** See lnet_md_t::options. */
#define LNET_MD_MAX_SIZE (1 << 7)
+/** See lnet_md_t::options. */
#define LNET_MD_KIOV (1 << 8)
/* For compatibility with Cray Portals */
#define LNET_MD_PHYS 0
+/** Infinite threshold on MD operations. See lnet_md_t::threshold */
#define LNET_MD_THRESH_INF (-1)
/* NB lustre portals uses struct iovec internally! */
typedef struct iovec lnet_md_iovec_t;
+/**
+ * A page-based fragment of a MD.
+ */
typedef struct {
+ /** Pointer to the page where the fragment resides */
cfs_page_t *kiov_page;
+ /** Length in bytes of the fragment */
unsigned int kiov_len;
+ /**
+ * Starting offset of the fragment within the page. Note that the
+ * end of the fragment must not pass the end of the page; i.e.,
+ * kiov_len + kiov_offset <= CFS_PAGE_SIZE.
+ */
unsigned int kiov_offset;
} lnet_kiov_t;
+/** @} lnet_md */
+/** \addtogroup lnet_eq
+ * @{ */
+
+/**
+ * Six types of events can be logged in an event queue.
+ */
typedef enum {
+ /** An incoming GET operation has completed on the MD. */
LNET_EVENT_GET,
+ /**
+ * An incoming PUT operation has completed on the MD. The
+ * underlying layers will not alter the memory (on behalf of this
+ * operation) once this event has been logged.
+ */
LNET_EVENT_PUT,
+ /**
+ * A REPLY operation has completed. This event is logged after the
+ * data (if any) from the REPLY has been written into the MD.
+ */
LNET_EVENT_REPLY,
+ /** An acknowledgment has been received. */
LNET_EVENT_ACK,
+ /**
+ * An outgoing send (PUT or GET) operation has completed. This event
+ * is logged after the entire buffer has been sent and it is safe for
+ * the caller to reuse the buffer.
+ *
+ * Note:
+ * - The LNET_EVENT_SEND doesn't guarantee message delivery. It can
+ * happen even when the message has not yet been put out on wire.
+ * - It's unsafe to assume that in an outgoing GET operation
+ * the LNET_EVENT_SEND event would happen before the
+ * LNET_EVENT_REPLY event. The same holds for LNET_EVENT_SEND and
+ * LNET_EVENT_ACK events in an outgoing PUT operation.
+ */
LNET_EVENT_SEND,
+ /**
+ * A MD has been unlinked. Note that LNetMDUnlink() does not
+ * necessarily trigger an LNET_EVENT_UNLINK event.
+ * \see LNetMDUnlink
+ */
LNET_EVENT_UNLINK,
} lnet_event_kind_t;
#ifdef __CYGWIN__
#pragma pack(push, 4)
#endif
+
+/**
+ * Information about an event on a MD.
+ */
typedef struct {
+ /** The identifier (nid, pid) of the target. */
lnet_process_id_t target;
+ /** The identifier (nid, pid) of the initiator. */
lnet_process_id_t initiator;
+ /**
+ * The NID of the immediate sender. If the request has been forwarded
+ * by routers, this is the NID of the last hop; otherwise it's the
+ * same as the initiator.
+ */
lnet_nid_t sender;
+ /** Indicates the type of the event. */
lnet_event_kind_t type;
+ /** The portal table index specified in the request */
unsigned int pt_index;
+ /** A copy of the match bits specified in the request. */
__u64 match_bits;
+ /** The length (in bytes) specified in the request. */
unsigned int rlength;
+ /**
+ * The length (in bytes) of the data that was manipulated by the
+ * operation. For truncated operations, the manipulated length will be
+ * the number of bytes specified by the MD (possibly with an offset,
+ * see lnet_md_t). For all other operations, the manipulated length
+ * will be the length of the requested operation, i.e. rlength.
+ */
unsigned int mlength;
+ /**
+ * The handle to the MD associated with the event. The handle may be
+ * invalid if the MD has been unlinked.
+ */
lnet_handle_md_t md_handle;
+ /**
+ * A snapshot of the state of the MD immediately after the event has
+ * been processed. In particular, the threshold field in md will
+ * reflect the value of the threshold after the operation occurred.
+ */
lnet_md_t md;
+ /**
+ * 64 bits of out-of-band user data. Only valid for LNET_EVENT_PUT.
+ * \see LNetPut
+ */
__u64 hdr_data;
+ /**
+ * Indicates the completion status of the operation. It's 0 for
+ * successful operations, otherwise it's an error code.
+ */
int status;
+ /**
+ * Indicates whether the MD has been unlinked. Note that:
+ * - An event with unlinked set is the last event on the MD.
+ * - This field is also set for an explicit LNET_EVENT_UNLINK event.
+ * \see LNetMDUnlink
+ */
int unlinked;
+ /**
+ * The displacement (in bytes) into the memory region that the
+ * operation used. The offset can be determined by the operation for
+ * a remote managed MD or by the local MD.
+ * \see lnet_md_t::options
+ */
unsigned int offset;
#ifdef CRAY_XT3
lnet_uid_t uid;
#endif
+ /**
+ * The sequence number for this event. Sequence numbers are unique
+ * to each event.
+ */
volatile lnet_seq_t sequence;
} lnet_event_t;
#ifdef __CYGWIN__
#pragma pop
#endif
+/**
+ * Event queue handler function type.
+ *
+ * The EQ handler runs for each event that is deposited into the EQ. The
+ * handler is supplied with a pointer to the event that triggered the
+ * handler invocation.
+ *
+ * The handler must not block, must be reentrant, and must not call any LNet
+ * API functions. It should return as quickly as possible.
+ */
+typedef void (*lnet_eq_handler_t)(lnet_event_t *event);
+#define LNET_EQ_HANDLER_NONE NULL
+/** @} lnet_eq */
+
+/** \addtogroup lnet_data
+ * @{ */
+
+/**
+ * Specify whether an acknowledgment should be sent by target when the PUT
+ * operation completes (i.e., when the data has been written to a MD of the
+ * target process).
+ *
+ * \see lnet_md_t::options for the discussion on LNET_MD_ACK_DISABLE by which
+ * acknowledgments can be disabled for a MD.
+ */
typedef enum {
+ /** Request an acknowledgment */
LNET_ACK_REQ,
+ /** Request that no acknowledgment should be generated. */
LNET_NOACK_REQ
} lnet_ack_req_t;
+/** @} lnet_data */
-typedef void (*lnet_eq_handler_t)(lnet_event_t *event);
-#define LNET_EQ_HANDLER_NONE NULL
-
+/** @} lnet */
#endif
}
#ifndef __KERNEL__
-/* Temporary workaround to allow uOSS and test programs force server
+/**
+ * Reserved API - do not use.
+ * Temporary workaround to allow uOSS and test programs force server
* mode in userspace. See comments near ln_server_mode_flag in
* lnet/lib-types.h */
return -ENETDOWN;
}
+/**
+ * Initialize LNet library.
+ *
+ * Only userspace program needs to call this function - it's automatically
+ * called in the kernel at module loading time. Caller has to call LNetFini()
+ * after a call to LNetInit(), if and only if the latter returned 0. It must
+ * be called exactly once.
+ *
+ * \return 0 on success, and -ve on failures.
+ */
int
LNetInit(void)
{
return 0;
}
+/**
+ * Finalize LNet library.
+ *
+ * Only userspace program needs to call this function. It can be called
+ * at most once.
+ *
+ * \pre LNetInit() called with success.
+ * \pre All LNet users called LNetNIFini() for matching LNetNIInit() calls.
+ */
void
LNetFini(void)
{
the_lnet.ln_init = 0;
}
+/**
+ * Set LNet PID and start LNet interfaces, routing, and forwarding.
+ *
+ * Userspace program should call this after a successful call to LNetInit().
+ * Users must call this function at least once before any other functions.
+ * For each successful call there must be a corresponding call to
+ * LNetNIFini(). For subsequent calls to LNetNIInit(), \a requested_pid is
+ * ignored.
+ *
+ * The PID used by LNet may be different from the one requested.
+ * See LNetGetId().
+ *
+ * \param requested_pid PID requested by the caller.
+ *
+ * \return 0 on success, and non-zero error code on failures.
+ */
int
LNetNIInit(lnet_pid_t requested_pid)
{
return rc;
}
+/**
+ * Stop LNet interfaces, routing, and forwarding.
+ *
+ * Users must call this function once for each successful call to LNetNIInit().
+ * Once the LNetNIFini() operation has been started, the results of pending
+ * API operations are undefined.
+ *
+ * \return always 0 for current implementation.
+ */
int
LNetNIFini()
{
return 0;
}
+/**
+ * This is an ugly hack to export IOC_LIBCFS_DEBUG_PEER and
+ * IOC_LIBCFS_PORTALS_COMPATIBILITY commands to users, by tweaking the LNet
+ * internal ioctl handler.
+ *
+ * IOC_LIBCFS_PORTALS_COMPATIBILITY is now deprecated, don't use it.
+ *
+ * \param cmd IOC_LIBCFS_DEBUG_PEER to print debugging data about a peer.
+ * The data will be printed to system console. Don't use it excessively.
+ * \param arg A pointer to lnet_process_id_t, process ID of the peer.
+ *
+ * \return Always return 0 when called by users directly (i.e., not via ioctl).
+ */
int
LNetCtl(unsigned int cmd, void *arg)
{
/* not reached */
}
+/**
+ * Retrieve the lnet_process_id_t ID of LNet interface at \a index. Note that
+ * all interfaces share a same PID, as requested by LNetNIInit().
+ *
+ * \param index Index of the interface to look up.
+ * \param id On successful return, this location will hold the
+ * lnet_process_id_t ID of the interface.
+ *
+ * \retval 0 If an interface exists at \a index.
+ * \retval -ENOENT If no interface has been found.
+ */
int
LNetGetId(unsigned int index, lnet_process_id_t *id)
{
return rc;
}
+/**
+ * Print a string representation of handle \a h into buffer \a str of
+ * \a len bytes.
+ */
void
LNetSnprintHandle(char *str, int len, lnet_handle_any_t h)
{
#define DEBUG_SUBSYSTEM S_LNET
#include <lnet/lib-lnet.h>
+/**
+ * Create an event queue that has room for \a count number of events.
+ *
+ * The event queue is circular and older events will be overwritten by new
+ * ones if they are not removed in time by the user using the functions
+ * LNetEQGet(), LNetEQWait(), or LNetEQPoll(). It is up to the user to
+ * determine the appropriate size of the event queue to prevent this loss
+ * of events. Note that when EQ handler is specified in \a callback, no
+ * event loss can happen, since the handler is run for each event deposited
+ * into the EQ.
+ *
+ * \param count The number of events to be stored in the event queue. It
+ * will be rounded up to the next power of two.
+ * \param callback A handler function that runs when an event is deposited
+ * into the EQ. The constant value LNET_EQ_HANDLER_NONE can be used to
+ * indicate that no event handler is desired.
+ * \param handle On successful return, this location will hold a handle for
+ * the newly created EQ.
+ *
+ * \retval 0 On success.
+ * \retval -EINVAL If an parameter is not valid.
+ * \retval -ENOMEM If memory for the EQ can't be allocated.
+ *
+ * \see lnet_eq_handler_t for the discussion on EQ handler semantics.
+ */
int
LNetEQAlloc(unsigned int count, lnet_eq_handler_t callback,
lnet_handle_eq_t *handle)
/* We need count to be a power of 2 so that when eq_{enq,deq}_seq
* overflow, they don't skip entries, so the queue has the same
- * apparant capacity at all times */
+ * apparent capacity at all times */
if (count != LOWEST_BIT_SET(count)) { /* not a power of 2 already */
do { /* knock off all but the top bit... */
return (0);
}
+/**
+ * Release the resources associated with an event queue if it's idle;
+ * otherwise do nothing and it's up to the user to try again.
+ *
+ * \param eqh A handle for the event queue to be released.
+ *
+ * \retval 0 If the EQ is not in use and freed.
+ * \retval -ENOENT If \a eqh does not point to a valid EQ.
+ * \retval -EBUSY If the EQ is still in use by some MDs.
+ */
int
LNetEQFree(lnet_handle_eq_t eqh)
{
RETURN(rc);
}
-
+/**
+ * A nonblocking function that can be used to get the next event in an EQ.
+ * If an event handler is associated with the EQ, the handler will run before
+ * this function returns successfully. The event is removed from the queue.
+ *
+ * \param eventq A handle for the event queue.
+ * \param event On successful return (1 or -EOVERFLOW), this location will
+ * hold the next event in the EQ.
+ *
+ * \retval 0 No pending event in the EQ.
+ * \retval 1 Indicates success.
+ * \retval -ENOENT If \a eventq does not point to a valid EQ.
+ * \retval -EOVERFLOW Indicates success (i.e., an event is returned) and that
+ * at least one event between this event and the last event obtained from the
+ * EQ has been dropped due to limited space in the EQ.
+ */
int
LNetEQGet (lnet_handle_eq_t eventq, lnet_event_t *event)
{
event, &which);
}
+/**
+ * Block the calling process until there is an event in the EQ.
+ * If an event handler is associated with the EQ, the handler will run before
+ * this function returns successfully. This function returns the next event
+ * in the EQ and removes it from the EQ.
+ *
+ * \param eventq A handle for the event queue.
+ * \param event On successful return (1 or -EOVERFLOW), this location will
+ * hold the next event in the EQ.
+ *
+ * \retval 1 Indicates success.
+ * \retval -ENOENT If \a eventq does not point to a valid EQ.
+ * \retval -EOVERFLOW Indicates success (i.e., an event is returned) and that
+ * at least one event between this event and the last event obtained from the
+ * EQ has been dropped due to limited space in the EQ.
+ */
int
LNetEQWait (lnet_handle_eq_t eventq, lnet_event_t *event)
{
event, &which);
}
+/**
+ * Block the calling process until there's an event from a set of EQs or
+ * timeout happens.
+ *
+ * If an event handler is associated with the EQ, the handler will run before
+ * this function returns successfully, in which case the corresponding event
+ * is consumed.
+ *
+ * LNetEQPoll() provides a timeout to allow applications to poll, block for a
+ * fixed period, or block indefinitely.
+ *
+ * \param eventqs,neq An array of EQ handles, and size of the array.
+ * \param timeout_ms Time in milliseconds to wait for an event to occur on
+ * one of the EQs. The constant LNET_TIME_FOREVER can be used to indicate an
+ * infinite timeout.
+ * \param event,which On successful return (1 or -EOVERFLOW), \a event will
+ * hold the next event in the EQs, and \a which will contain the index of the
+ * EQ from which the event was taken.
+ *
+ * \retval 0 No pending event in the EQs after timeout.
+ * \retval 1 Indicates success.
+ * \retval -EOVERFLOW Indicates success (i.e., an event is returned) and that
+ * at least one event between this event and the last event obtained from the
+ * EQ indicated by \a which has been dropped due to limited space in the EQ.
+ * \retval -ENOENT If there's an invalid handle in \a eventqs.
+ */
int
LNetEQPoll (lnet_handle_eq_t *eventqs, int neq, int timeout_ms,
lnet_event_t *event, int *which)
return 0;
}
+/**
+ * Create a memory descriptor and attach it to a ME
+ *
+ * \param meh A handle for a ME to associate the new MD with.
+ * \param umd Provides initial values for the user-visible parts of a MD.
+ * Other than its use for initialization, there is no linkage between this
+ * structure and the MD maintained by the LNet.
+ * \param unlink A flag to indicate whether the MD is automatically unlinked
+ * when it becomes inactive, either because the operation threshold drops to
+ * zero or because the available memory becomes less than \a umd.max_size.
+ * (Note that the check for unlinking a MD only occurs after the completion
+ * of a successful operation on the MD.) The value LNET_UNLINK enables auto
+ * unlinking; the value LNET_RETAIN disables it.
+ * \param handle On successful returns, a handle to the newly created MD is
+ * saved here. This handle can be used later in LNetMDUnlink().
+ *
+ * \retval 0 On success.
+ * \retval -EINVAL If \a umd is not valid.
+ * \retval -ENOMEM If new MD cannot be allocated.
+ * \retval -ENOENT Either \a meh or \a umd.eq_handle does not point to a
+ * valid object. Note that it's OK to supply a NULL \a umd.eq_handle by
+ * calling LNetInvalidateHandle() on it.
+ * \retval -EBUSY If the ME pointed to by \a meh is already associated with
+ * a MD.
+ */
int
LNetMDAttach(lnet_handle_me_t meh, lnet_md_t umd,
lnet_unlink_t unlink, lnet_handle_md_t *handle)
return (rc);
}
+/**
+ * Create a "free floating" memory descriptor - a MD that is not associated
+ * with a ME. Such MDs are usually used in LNetPut() and LNetGet() operations.
+ *
+ * \param umd,unlink See the discussion for LNetMDAttach().
+ * \param handle On successful returns, a handle to the newly created MD is
+ * saved here. This handle can be used later in LNetMDUnlink(), LNetPut(),
+ * and LNetGet() operations.
+ *
+ * \retval 0 On success.
+ * \retval -EINVAL If \a umd is not valid.
+ * \retval -ENOMEM If new MD cannot be allocated.
+ * \retval -ENOENT \a umd.eq_handle does not point to a valid EQ. Note that
+ * it's OK to supply a NULL \a umd.eq_handle by calling
+ * LNetInvalidateHandle() on it.
+ */
int
LNetMDBind(lnet_md_t umd, lnet_unlink_t unlink, lnet_handle_md_t *handle)
{
return (rc);
}
+/**
+ * Unlink the memory descriptor from any ME it may be linked to and release
+ * the internal resources associated with it.
+ *
+ * This function does not free the memory region associated with the MD;
+ * i.e., the memory the user allocated for this MD. If the ME associated with
+ * this MD is not NULL and was created with auto unlink enabled, the ME is
+ * unlinked as well (see LNetMEAttach()).
+ *
+ * Explicitly unlinking a MD via this function call has the same behavior as
+ * a MD that has been automatically unlinked, except that no LNET_EVENT_UNLINK
+ * is generated in the latter case.
+ *
+ * An unlinked event can be reported in two ways:
+ * - If there's no pending operations on the MD, it's unlinked immediately
+ * and an LNET_EVENT_UNLINK event is logged before this function returns.
+ * - Otherwise, the MD is only marked for deletion when this function
+ * returns, and the unlinked event will be piggybacked on the event of
+ * the completion of the last operation by setting the unlinked field of
+ * the event. No dedicated LNET_EVENT_UNLINK event is generated.
+ *
+ * Note that in both cases the unlinked field of the event is always set; no
+ * more event will happen on the MD after such an event is logged.
+ *
+ * \param mdh A handle for the MD to be unlinked.
+ *
+ * \retval 0 On success.
+ * \retval -ENOENT If \a mdh does not point to a valid MD object.
+ */
int
LNetMDUnlink (lnet_handle_md_t mdh)
{
return 0;
}
+/**
+ * Create and attach a match entry to the match list of \a portal. The new
+ * ME is empty, i.e. not associated with a memory descriptor. LNetMDAttach()
+ * can be used to attach a MD to an empty ME.
+ *
+ * \param portal The portal table index where the ME should be attached.
+ * \param match_id Specifies the match criteria for the process ID of
+ * the requester. The constants LNET_PID_ANY and LNET_NID_ANY can be
+ * used to wildcard either of the identifiers in the lnet_process_id_t
+ * structure.
+ * \param match_bits,ignore_bits Specify the match criteria to apply
+ * to the match bits in the incoming request. The ignore bits are used
+ * to mask out insignificant bits in the incoming match bits. The resulting
+ * bits are then compared to the ME's match bits to determine if the
+ * incoming request meets the match criteria.
+ * \param unlink Indicates whether the ME should be unlinked when the memory
+ * descriptor associated with it is unlinked (Note that the check for
+ * unlinking a ME only occurs when the memory descriptor is unlinked.).
+ * Valid values are LNET_RETAIN and LNET_UNLINK.
+ * \param pos Indicates whether the new ME should be prepended or
+ * appended to the match list. Allowed constants: LNET_INS_BEFORE,
+ * LNET_INS_AFTER.
+ * \param handle On successful returns, a handle to the newly created ME
+ * object is saved here. This handle can be used later in LNetMEInsert(),
+ * LNetMEUnlink(), or LNetMDAttach() functions.
+ *
+ * \retval 0 On success.
+ * \retval -EINVAL If \a portal is invalid.
+ * \retval -ENOMEM If new ME object cannot be allocated.
+ */
int
LNetMEAttach(unsigned int portal,
lnet_process_id_t match_id,
return 0;
}
+/**
+ * Create and a match entry and insert it before or after the ME pointed to by
+ * \a current_meh. The new ME is empty, i.e. not associated with a memory
+ * descriptor. LNetMDAttach() can be used to attach a MD to an empty ME.
+ *
+ * This function is identical to LNetMEAttach() except for the position
+ * where the new ME is inserted.
+ *
+ * \param current_meh A handle for a ME. The new ME will be inserted
+ * immediately before or immediately after this ME.
+ * \param match_id,match_bits,ignore_bits,unlink,pos,handle See the discussion
+ * for LNetMEAttach().
+ *
+ * \retval 0 On success.
+ * \retval -ENOMEM If new ME object cannot be allocated.
+ * \retval -ENOENT If \a current_meh does not point to a valid match entry.
+ */
int
LNetMEInsert(lnet_handle_me_t current_meh,
lnet_process_id_t match_id,
return 0;
}
+/**
+ * Unlink a match entry from its match list.
+ *
+ * This operation also releases any resources associated with the ME. If a
+ * memory descriptor is attached to the ME, then it will be unlinked as well
+ * and an unlink event will be generated. It is an error to use the ME handle
+ * after calling LNetMEUnlink().
+ *
+ * \param meh A handle for the ME to be unlinked.
+ *
+ * \retval 0 On success.
+ * \retval -ENOENT If \a meh does not point to a valid ME.
+ * \see LNetMDUnlink() for the discussion on delivering unlink event.
+ */
int
LNetMEUnlink(lnet_handle_me_t meh)
{
LNET_UNLOCK();
}
+/**
+ * Turn on the lazy portal attribute. Use with caution!
+ *
+ * This portal attribute only affects incoming PUT requests to the portal,
+ * and is off by default. By default, if there's no matching MD for an
+ * incoming PUT request, it is simply dropped. With the lazy attribute on,
+ * such requests are queued indefinitely until either a matching MD is
+ * posted to the portal or the lazy attribute is turned off.
+ *
+ * It would prevent dropped requests, however it should be regarded as the
+ * last line of defense - i.e. users must keep a close watch on active
+ * buffers on a lazy portal and once it becomes too low post more buffers as
+ * soon as possible. This is because delayed requests usually have detrimental
+ * effects on underlying network connections. A few delayed requests often
+ * suffice to bring an underlying connection to a complete halt, due to flow
+ * control mechanisms.
+ *
+ * There's also a DOS attack risk. If users don't post match-all MDs on a
+ * lazy portal, a malicious peer can easily stop a service by sending some
+ * PUT requests with match bits that won't match any MD. A routed server is
+ * especially vulnerable since the connections to its neighbor routers are
+ * shared among all clients.
+ *
+ * \param portal Index of the portal to enable the lazy attribute on.
+ *
+ * \retval 0 On success.
+ * \retval -EINVAL If \a portal is not a valid index.
+ */
int
LNetSetLazyPortal(int portal)
{
return 0;
}
+/**
+ * Turn off the lazy portal attribute. Delayed requests on the portal,
+ * if any, will be all dropped when this function returns.
+ *
+ * \param portal Index of the portal to disable the lazy attribute on.
+ *
+ * \retval 0 On success.
+ * \retval -EINVAL If \a portal is not a valid index.
+ */
int
LNetClearLazyPortal(int portal)
{
return 0;
}
+/**
+ * Initiate an asynchronous PUT operation.
+ *
+ * There are several events associated with a PUT: completion of the send on
+ * the initiator node (LNET_EVENT_SEND), and when the send completes
+ * successfully, the receipt of an acknowledgment (LNET_EVENT_ACK) indicating
+ * that the operation was accepted by the target. The event LNET_EVENT_PUT is
+ * used at the target node to indicate the completion of incoming data
+ * delivery.
+ *
+ * The local events will be logged in the EQ associated with the MD pointed to
+ * by \a mdh handle. Using a MD without an associated EQ results in these
+ * events being discarded. In this case, the caller must have another
+ * mechanism (e.g., a higher level protocol) for determining when it is safe
+ * to modify the memory region associated with the MD.
+ *
+ * Note that LNet does not guarantee the order of LNET_EVENT_SEND and
+ * LNET_EVENT_ACK, though intuitively ACK should happen after SEND.
+ *
+ * \param self Indicates the NID of a local interface through which to send
+ * the PUT request. Use LNET_NID_ANY to let LNet choose one by itself.
+ * \param mdh A handle for the MD that describes the memory to be sent. The MD
+ * must be "free floating" (See LNetMDBind()).
+ * \param ack Controls whether an acknowledgment is requested.
+ * Acknowledgments are only sent when they are requested by the initiating
+ * process and the target MD enables them.
+ * \param target A process identifier for the target process.
+ * \param portal The index in the \a target's portal table.
+ * \param match_bits The match bits to use for MD selection at the target
+ * process.
+ * \param offset The offset into the target MD (only used when the target
+ * MD has the LNET_MD_MANAGE_REMOTE option set).
+ * \param hdr_data 64 bits of user data that can be included in the message
+ * header. This data is written to an event queue entry at the target if an
+ * EQ is present on the matching MD.
+ *
+ * \retval 0 Success, and only in this case events will be generated
+ * and logged to EQ (if it exists).
+ * \retval -EIO Simulated failure.
+ * \retval -ENOMEM Memory allocation failure.
+ * \retval -ENOENT Invalid MD object.
+ *
+ * \see lnet_event_t::hdr_data and lnet_event_kind_t.
+ */
int
LNetPut(lnet_nid_t self, lnet_handle_md_t mdh, lnet_ack_req_t ack,
lnet_process_id_t target, unsigned int portal,
reply->msg_ev.mlength = len;
}
+/**
+ * Initiate an asynchronous GET operation.
+ *
+ * On the initiator node, an LNET_EVENT_SEND is logged when the GET request
+ * is sent, and an LNET_EVENT_REPLY is logged when the data returned from
+ * the target node in the REPLY has been written to local MD.
+ *
+ * On the target node, an LNET_EVENT_GET is logged when the GET request
+ * arrives and is accepted into a MD.
+ *
+ * \param self,target,portal,match_bits,offset See the discussion in LNetPut().
+ * \param mdh A handle for the MD that describes the memory into which the
+ * requested data will be received. The MD must be "free floating" (See LNetMDBind()).
+ *
+ * \retval 0 Success, and only in this case events will be generated
+ * and logged to EQ (if it exists) of the MD.
+ * \retval -EIO Simulated failure.
+ * \retval -ENOMEM Memory allocation failure.
+ * \retval -ENOENT Invalid MD object.
+ */
int
LNetGet(lnet_nid_t self, lnet_handle_md_t mdh,
lnet_process_id_t target, unsigned int portal,
return 0;
}
+/**
+ * Calculate distance to node at \a dstnid.
+ *
+ * \param dstnid Target NID.
+ * \param srcnidp If not NULL, NID of the local interface to reach \a dstnid
+ * is saved here.
+ * \param orderp If not NULL, order of the route to reach \a dstnid is saved
+ * here.
+ *
+ * \retval 0 If \a dstnid belongs to a local interface, and reserved option
+ * local_nid_dist_zero is set, which is the default.
+ * \retval positives Distance to target NID, i.e. number of hops plus one.
+ * \retval -EHOSTUNREACH If \a dstnid is not reachable.
+ */
int
LNetDist (lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp)
{
return -EHOSTUNREACH;
}
+/**
+ * Set the number of asynchronous messages expected from a target process.
+ *
+ * This function is only meaningful for userspace callers. It's a no-op when
+ * called from kernel.
+ *
+ * Asynchronous messages are those that can come from a target when the
+ * userspace process is not waiting for IO to complete; e.g., AST callbacks
+ * from Lustre servers. Specifying the expected number of such messages
+ * allows them to be eagerly received when user process is not running in
+ * LNet; otherwise network errors may occur.
+ *
+ * \param id Process ID of the target process.
+ * \param nasync Number of asynchronous messages expected from the target.
+ *
+ * \return 0 on success, and an error code otherwise.
+ */
int
LNetSetAsync(lnet_process_id_t id, int nasync)
{
int rc2;
/* Target on a local network? */
-
ni = lnet_net2ni(LNET_NIDNET(id.nid));
if (ni != NULL) {
if (ni->ni_lnd->lnd_setasync != NULL)
/* There is no race since both event consumers and event producers
* take the LNET_LOCK, so we don't screw around with memory
- * barriers, setting the sequence number last or wierd structure
+ * barriers, setting the sequence number last or weird structure
* layout assertions. */
*eq_slot = *ev;