From 8d8d0f2cacd5769a2e324fbd37f5a0674748621f Mon Sep 17 00:00:00 2001 From: Isaac Huang Date: Wed, 18 Aug 2010 09:40:56 +0400 Subject: [PATCH 1/1] b=20973 Doxygen comments for LNet API Doxygen comments for LNet API. This is a comment-only change. i=maxim --- build/doxyfile.api | 11 +- build/doxyfile.ref | 11 +- libcfs/include/libcfs/libcfs_private.h | 6 + libcfs/libcfs/watchdog.c | 2 +- lnet/include/lnet/api.h | 123 ++++++++++--- lnet/include/lnet/types.h | 314 ++++++++++++++++++++++++++++++++- lnet/lnet/api-ni.c | 76 +++++++- lnet/lnet/lib-eq.c | 96 +++++++++- lnet/lnet/lib-md.c | 70 ++++++++ lnet/lnet/lib-me.c | 61 +++++++ lnet/lnet/lib-move.c | 133 +++++++++++++- lnet/lnet/lib-msg.c | 2 +- 12 files changed, 868 insertions(+), 37 deletions(-) diff --git a/build/doxyfile.api b/build/doxyfile.api index e871875..9df11fc 100644 --- a/build/doxyfile.api +++ b/build/doxyfile.api @@ -103,9 +103,16 @@ INPUT_ENCODING = UTF-8 FILE_PATTERNS = *.h \ *.c RECURSIVE = YES -EXCLUDE = +EXCLUDE = lnet/klnds/ciblnd/ \ + lnet/klnds/gmlnd/ \ + lnet/klnds/iiblnd/ \ + lnet/klnds/mxlnd/ \ + lnet/klnds/openiblnd/ \ + lnet/klnds/qswlnd/ \ + lnet/klnds/ralnd/ \ + lnet/klnds/viblnd/ EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = +EXCLUDE_PATTERNS = */darwin/* EXCLUDE_SYMBOLS = EXPORT_SYMBOL* EXAMPLE_PATH = EXAMPLE_PATTERNS = * diff --git a/build/doxyfile.ref b/build/doxyfile.ref index ab1445f..4b0602a 100644 --- a/build/doxyfile.ref +++ b/build/doxyfile.ref @@ -103,9 +103,16 @@ INPUT_ENCODING = UTF-8 FILE_PATTERNS = *.h \ *.c RECURSIVE = YES -EXCLUDE = +EXCLUDE = lnet/klnds/ciblnd/ \ + lnet/klnds/gmlnd/ \ + lnet/klnds/iiblnd/ \ + lnet/klnds/mxlnd/ \ + lnet/klnds/openiblnd/ \ + lnet/klnds/qswlnd/ \ + lnet/klnds/ralnd/ \ + lnet/klnds/viblnd/ EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = +EXCLUDE_PATTERNS = */darwin/* EXCLUDE_SYMBOLS = EXPORT_SYMBOL* EXAMPLE_PATH = EXAMPLE_PATTERNS = * diff --git a/libcfs/include/libcfs/libcfs_private.h b/libcfs/include/libcfs/libcfs_private.h index a859e88..51dc986 100644 --- a/libcfs/include/libcfs/libcfs_private.h +++ b/libcfs/include/libcfs/libcfs_private.h @@ -324,14 +324,20 @@ void cfs_free_nidlist(cfs_list_t *list); int cfs_parse_nidlist(char *str, int len, cfs_list_t *list); int cfs_match_nid(lnet_nid_t nid, cfs_list_t *list); +/** \addtogroup lnet_addr + * @{ */ /* how an LNET NID encodes net:address */ +/** extract the address part of an lnet_nid_t */ #define LNET_NIDADDR(nid) ((__u32)((nid) & 0xffffffff)) +/** extract the network part of an lnet_nid_t */ #define LNET_NIDNET(nid) ((__u32)(((nid) >> 32)) & 0xffffffff) +/** make an lnet_nid_t from a network part and an address part */ #define LNET_MKNID(net,addr) ((((__u64)(net))<<32)|((__u64)(addr))) /* how net encodes type:number */ #define LNET_NETNUM(net) ((net) & 0xffff) #define LNET_NETTYP(net) (((net) >> 16) & 0xffff) #define LNET_MKNET(typ,num) ((((__u32)(typ))<<16)|((__u32)(num))) +/** @} lnet_addr */ /* max value for numeric network address */ #define MAX_NUMERIC_VALUE 0xffffffff diff --git a/libcfs/libcfs/watchdog.c b/libcfs/libcfs/watchdog.c index 274c59b..423a4bd 100644 --- a/libcfs/libcfs/watchdog.c +++ b/libcfs/libcfs/watchdog.c @@ -83,7 +83,7 @@ static unsigned long lcw_flags = 0; /* * Number of outstanding watchdogs. * When it hits 1, we start the dispatcher. - * When it hits 0, we stop the distpatcher. + * When it hits 0, we stop the dispatcher. */ static __u32 lcw_refcount = 0; static CFS_DECLARE_MUTEX(lcw_refcount_sem); diff --git a/lnet/include/lnet/api.h b/lnet/include/lnet/api.h index 0d5c7f0..9d1c49c 100644 --- a/lnet/include/lnet/api.h +++ b/lnet/include/lnet/api.h @@ -37,28 +37,68 @@ #ifndef __LNET_API_H__ #define __LNET_API_H__ +/** \defgroup lnet LNet + * + * The Lustre Networking subsystem. + * + * LNet is an asynchronous message-passing API, which provides an unreliable + * connectionless service that can't guarantee any order. It supports OFA IB, + * TCP/IP, and Cray Portals, and routes between heterogeneous networks. + * + * LNet can run both in OS kernel space and in userspace as a library. + * @{ + */ + #include +/** \defgroup lnet_init_fini Initialization and cleanup + * The LNet must be properly initialized before any LNet calls can be made. + * @{ */ int LNetInit(void); void LNetFini(void); int LNetNIInit(lnet_pid_t requested_pid); int LNetNIFini(void); +/** @} lnet_init_fini */ +/** \defgroup lnet_addr LNet addressing and basic types + * + * Addressing scheme and basic data types of LNet. + * + * The LNet API is memory-oriented, so LNet must be able to address not only + * end-points but also memory region within a process address space. + * An ::lnet_nid_t addresses an end-point. An ::lnet_pid_t identifies a process + * in a node. A portal represents an opening in the address space of a + * process. Match bits is criteria to identify a region of memory inside a + * portal, and offset specifies an offset within the memory region. + * + * LNet creates a table of portals for each process during initialization. + * This table has MAX_PORTALS entries and its size can't be dynamically + * changed. A portal stays empty until the owning process starts to add + * memory regions to it. A portal is sometimes called an index because + * it's an entry in the portals table of a process. + * + * \see LNetMEAttach + * @{ */ int LNetGetId(unsigned int index, lnet_process_id_t *id); int LNetDist(lnet_nid_t nid, lnet_nid_t *srcnid, __u32 *order); -int LNetCtl(unsigned int cmd, void *arg); -void LNetSnprintHandle (char *str, int str_len, lnet_handle_any_t handle); +void LNetSnprintHandle(char *str, int str_len, lnet_handle_any_t handle); -/* - * Portals - */ -int LNetSetLazyPortal(int portal); -int LNetClearLazyPortal(int portal); +/** @} lnet_addr */ -/* - * Match entries - */ + +/** \defgroup lnet_me Match entries + * + * A match entry (abbreviated as ME) describes a set of criteria to accept + * incoming requests. + * + * A portal is essentially a match list plus a set of attributes. A match + * list is a chain of MEs. Each ME includes a pointer to a memory descriptor + * and a set of match criteria. The match criteria can be used to reject + * incoming requests based on process ID or the match bits provided in the + * request. MEs can be dynamically inserted into a match list by LNetMEAttach() + * and LNetMEInsert(), and removed from its list by LNetMEUnlink(). + * @{ */ int LNetMEAttach(unsigned int portal, lnet_process_id_t match_id_in, __u64 match_bits_in, @@ -76,10 +116,20 @@ int LNetMEInsert(lnet_handle_me_t current_in, lnet_handle_me_t *handle_out); int LNetMEUnlink(lnet_handle_me_t current_in); +/** @} lnet_me */ -/* - * Memory descriptors - */ +/** \defgroup lnet_md Memory descriptors + * + * A memory descriptor contains information about a region of a user's + * memory (either in kernel or user space) and optionally points to an + * event queue where information about the operations performed on the + * memory descriptor are recorded. Memory descriptor is abbreviated as + * MD and can be used interchangeably with the memory region it describes. + * + * The LNet API provides two operations to create MDs: LNetMDAttach() + * and LNetMDBind(); one operation to unlink and release the resources + * associated with a MD: LNetMDUnlink(). + * @{ */ int LNetMDAttach(lnet_handle_me_t current_in, lnet_md_t md_in, lnet_unlink_t unlink_in, @@ -90,10 +140,30 @@ int LNetMDBind(lnet_md_t md_in, lnet_handle_md_t *handle_out); int LNetMDUnlink(lnet_handle_md_t md_in); +/** @} lnet_md */ -/* - * Event queues - */ +/** \defgroup lnet_eq Events and event queues + * + * Event queues (abbreviated as EQ) are used to log operations performed on + * local MDs. In particular, they signal the completion of a data transmission + * into or out of a MD. They can also be used to hold acknowledgments for + * completed PUT operations and indicate when a MD has been unlinked. Multiple + * MDs can share a single EQ. An EQ may have an optional event handler + * associated with it. If an event handler exists, it will be run for each + * event that is deposited into the EQ. + * + * In addition to the lnet_handle_eq_t, the LNet API defines two types + * associated with events: The ::lnet_event_kind_t defines the kinds of events + * that can be stored in an EQ. The lnet_event_t defines a structure that + * holds the information about with an event. + * + * There are five functions for dealing with EQs: LNetEQAlloc() is used to + * create an EQ and allocate the resources needed, while LNetEQFree() + * releases these resources and free the EQ. LNetEQGet() retrieves the next + * event from an EQ, and LNetEQWait() can be used to block a process until + * an EQ has at least one event. LNetEQPoll() can be used to test or wait + * on multiple EQs. + * @{ */ int LNetEQAlloc(unsigned int count_in, lnet_eq_handler_t handler, lnet_handle_eq_t *handle_out); @@ -112,10 +182,13 @@ int LNetEQPoll(lnet_handle_eq_t *eventqs_in, int timeout_ms, lnet_event_t *event_out, int *which_eq_out); +/** @} lnet_eq */ -/* - * Data movement - */ +/** \defgroup lnet_data Data movement operations + * + * The LNet API provides two data movement operations: LNetPut() + * and LNetGet(). + * @{ */ int LNetPut(lnet_nid_t self, lnet_handle_md_t md_in, lnet_ack_req_t ack_req_in, @@ -131,8 +204,16 @@ int LNetGet(lnet_nid_t self, unsigned int portal_in, __u64 match_bits_in, unsigned int offset_in); +/** @} lnet_data */ + +/** \defgroup lnet_misc Miscellaneous operations. + * Miscellaneous operations. + * @{ */ +int LNetSetLazyPortal(int portal); +int LNetClearLazyPortal(int portal); +int LNetCtl(unsigned int cmd, void *arg); int LNetSetAsync(lnet_process_id_t id, int nasync); #ifndef __KERNEL__ @@ -141,6 +222,8 @@ int LNetSetAsync(lnet_process_id_t id, int nasync); * lnet/lib-types.h */ void lnet_server_mode(); -#endif +#endif +/** @} lnet_misc */ +/** @} lnet */ #endif diff --git a/lnet/include/lnet/types.h b/lnet/include/lnet/types.h index e971548..bb6a5f0 100644 --- a/lnet/include/lnet/types.h +++ b/lnet/include/lnet/types.h @@ -37,14 +37,39 @@ #ifndef __LNET_TYPES_H__ #define __LNET_TYPES_H__ +/** \addtogroup lnet + * @{ */ + #include -#define LNET_RESERVED_PORTAL 0 /* portals reserved for lnet's own use */ +/** \addtogroup lnet_addr + * @{ */ + +/** Portal reserved for LNet's own use. + * \see lustre/include/lustre/lustre_idl.h for Lustre portal assignments. + */ +#define LNET_RESERVED_PORTAL 0 +/** + * Address of an end-point in an LNet network. + * + * A node can have multiple end-points and hence multiple addresses. + * An LNet network can be a simple network (e.g. tcp0) or a network of + * LNet networks connected by LNet routers. Therefore an end-point address + * has two parts: network ID, and address within a network. + * + * \see LNET_NIDNET, LNET_NIDADDR, and LNET_MKNID. + */ typedef __u64 lnet_nid_t; +/** + * ID of a process in a node. Shortened as PID to distinguish from + * lnet_process_id_t, the global process ID. + */ typedef __u32 lnet_pid_t; +/** wildcard NID that matches any end-point address */ #define LNET_NID_ANY ((lnet_nid_t) -1) +/** wildcard PID that matches any lnet_pid_t */ #define LNET_PID_ANY ((lnet_pid_t) -1) #ifdef CRAY_XT3 @@ -57,6 +82,16 @@ typedef __u32 lnet_uid_t; #define LNET_TIME_FOREVER (-1) +/** + * Objects maintained by the LNet are accessed through handles. Handle types + * have names of the form lnet_handle_xx_t, where xx is one of the two letter + * object type codes ('eq' for event queue, 'md' for memory descriptor, and + * 'me' for match entry). + * Each type of object is given a unique handle type to enhance type checking. + * The type lnet_handle_any_t can be used when a generic handle is needed. + * Every handle value can be converted into a value of type lnet_handle_any_t + * without loss of information. + */ typedef struct { __u64 cookie; } lnet_handle_any_t; @@ -67,43 +102,171 @@ typedef lnet_handle_any_t lnet_handle_me_t; #define LNET_WIRE_HANDLE_COOKIE_NONE (-1) +/** + * Invalidate handle \a h. + */ static inline void LNetInvalidateHandle(lnet_handle_any_t *h) { h->cookie = LNET_WIRE_HANDLE_COOKIE_NONE; } +/** + * Compare handles \a h1 and \a h2. + * + * \return 1 if handles are equal, 0 if otherwise. + */ static inline int LNetHandleIsEqual (lnet_handle_any_t h1, lnet_handle_any_t h2) { return (h1.cookie == h2.cookie); } +/** + * Check whether handle \a h is invalid. + * + * \return 1 if handle is invalid, 0 if valid. + */ static inline int LNetHandleIsInvalid(lnet_handle_any_t h) { return (LNET_WIRE_HANDLE_COOKIE_NONE == h.cookie); } +/** + * Global process ID. + */ typedef struct { + /** node id */ lnet_nid_t nid; - lnet_pid_t pid; /* node id / process id */ + /** process id */ + lnet_pid_t pid; } lnet_process_id_t; +/** @} lnet_addr */ + +/** \addtogroup lnet_me + * @{ */ +/** + * Specifies whether the match entry or memory descriptor should be unlinked + * automatically (LNET_UNLINK) or not (LNET_RETAIN). + */ typedef enum { LNET_RETAIN = 0, LNET_UNLINK } lnet_unlink_t; +/** + * Values of the type lnet_ins_pos_t are used to control where a new match + * entry is inserted. The value LNET_INS_BEFORE is used to insert the new + * entry before the current entry or before the head of the list. The value + * LNET_INS_AFTER is used to insert the new entry after the current entry + * or after the last item in the list. + */ typedef enum { LNET_INS_BEFORE, LNET_INS_AFTER } lnet_ins_pos_t; +/** @} lnet_me */ + +/** \addtogroup lnet_md + * @{ */ + +/** + * Defines the visible parts of a memory descriptor. Values of this type + * are used to initialize memory descriptors. + */ typedef struct { + /** + * Specify the memory region associated with the memory descriptor. + * If the options field has: + * - LNET_MD_KIOV bit set: The start field points to the starting + * address of an array of lnet_kiov_t and the length field specifies + * the number of entries in the array. The length can't be bigger + * than LNET_MAX_IOV. The lnet_kiov_t is used to describe page-based + * fragments that are not necessarily mapped in virtal memory. + * - LNET_MD_IOVEC bit set: The start field points to the starting + * address of an array of struct iovec and the length field specifies + * the number of entries in the array. The length can't be bigger + * than LNET_MAX_IOV. The struct iovec is used to describe fragments + * that have virtual addresses. + * - Otherwise: The memory region is contiguous. The start field + * specifies the starting address for the memory region and the + * length field specifies its length. + * + * When the memory region is fragmented, all fragments but the first + * one must start on page boundary, and all but the last must end on + * page boundary. + */ void *start; unsigned int length; + /** + * Specifies the maximum number of operations that can be performed + * on the memory descriptor. An operation is any action that could + * possibly generate an event. In the usual case, the threshold value + * is decremented for each operation on the MD. When the threshold + * drops to zero, the MD becomes inactive and does not respond to + * operations. A threshold value of LNET_MD_THRESH_INF indicates that + * there is no bound on the number of operations that may be applied + * to a MD. + */ int threshold; + /** + * Specifies the largest incoming request that the memory descriptor + * should respond to. When the unused portion of a MD (length - + * local offset) falls below this value, the MD becomes inactive and + * does not respond to further operations. This value is only used + * if the LNET_MD_MAX_SIZE option is set. + */ int max_size; + /** + * Specifies the behavior of the memory descriptor. A bitwise OR + * of the following values can be used: + * - LNET_MD_OP_PUT: The LNet PUT operation is allowed on this MD. + * - LNET_MD_OP_GET: The LNet GET operation is allowed on this MD. + * - LNET_MD_MANAGE_REMOTE: The offset used in accessing the memory + * region is provided by the incoming request. By default, the + * offset is maintained locally. When maintained locally, the + * offset is incremented by the length of the request so that + * the next operation (PUT or GET) will access the next part of + * the memory region. Note that only one offset variable exists + * per memory descriptor. If both PUT and GET operations are + * performed on a memory descriptor, the offset is updated each time. + * - LNET_MD_TRUNCATE: The length provided in the incoming request can + * be reduced to match the memory available in the region (determined + * by subtracting the offset from the length of the memory region). + * By default, if the length in the incoming operation is greater + * than the amount of memory available, the operation is rejected. + * - LNET_MD_ACK_DISABLE: An acknowledgment should not be sent for + * incoming PUT operations, even if requested. By default, + * acknowledgments are sent for PUT operations that request an + * acknowledgment. Acknowledgments are never sent for GET operations. + * The data sent in the REPLY serves as an implicit acknowledgment. + * - LNET_MD_KIOV: The start and length fields specify an array of + * lnet_kiov_t. + * - LNET_MD_IOVEC: The start and length fields specify an array of + * struct iovec. + * - LNET_MD_MAX_SIZE: The max_size field is valid. + * + * Note: + * - LNET_MD_KIOV or LNET_MD_IOVEC allows for a scatter/gather + * capability for memory descriptors. They can't be both set. + * - When LNET_MD_MAX_SIZE is set, the total length of the memory + * region (i.e. sum of all fragment lengths) must not be less than + * \a max_size. + */ unsigned int options; + /** + * A user-specified value that is associated with the memory + * descriptor. The value does not need to be a pointer, but must fit + * in the space used by a pointer. This value is recorded in events + * associated with operations on this MD. + */ void *user_ptr; + /** + * A handle for the event queue used to log the operations performed on + * the memory region. If this argument is a NULL handle (i.e. nullified + * by LNetInvalidateHandle()), operations performed on this memory + * descriptor are not logged. + */ lnet_handle_eq_t eq_handle; } lnet_md_t; @@ -111,7 +274,7 @@ typedef struct { #define LNET_MTU_BITS 20 #define LNET_MTU (1< +/** + * Create an event queue that has room for \a count number of events. + * + * The event queue is circular and older events will be overwritten by new + * ones if they are not removed in time by the user using the functions + * LNetEQGet(), LNetEQWait(), or LNetEQPoll(). It is up to the user to + * determine the appropriate size of the event queue to prevent this loss + * of events. Note that when EQ handler is specified in \a callback, no + * event loss can happen, since the handler is run for each event deposited + * into the EQ. + * + * \param count The number of events to be stored in the event queue. It + * will be rounded up to the next power of two. + * \param callback A handler function that runs when an event is deposited + * into the EQ. The constant value LNET_EQ_HANDLER_NONE can be used to + * indicate that no event handler is desired. + * \param handle On successful return, this location will hold a handle for + * the newly created EQ. + * + * \retval 0 On success. + * \retval -EINVAL If an parameter is not valid. + * \retval -ENOMEM If memory for the EQ can't be allocated. + * + * \see lnet_eq_handler_t for the discussion on EQ handler semantics. + */ int LNetEQAlloc(unsigned int count, lnet_eq_handler_t callback, lnet_handle_eq_t *handle) @@ -52,7 +77,7 @@ LNetEQAlloc(unsigned int count, lnet_eq_handler_t callback, /* We need count to be a power of 2 so that when eq_{enq,deq}_seq * overflow, they don't skip entries, so the queue has the same - * apparant capacity at all times */ + * apparent capacity at all times */ if (count != LOWEST_BIT_SET(count)) { /* not a power of 2 already */ do { /* knock off all but the top bit... */ @@ -99,6 +124,16 @@ LNetEQAlloc(unsigned int count, lnet_eq_handler_t callback, return (0); } +/** + * Release the resources associated with an event queue if it's idle; + * otherwise do nothing and it's up to the user to try again. + * + * \param eqh A handle for the event queue to be released. + * + * \retval 0 If the EQ is not in use and freed. + * \retval -ENOENT If \a eqh does not point to a valid EQ. + * \retval -EBUSY If the EQ is still in use by some MDs. + */ int LNetEQFree(lnet_handle_eq_t eqh) { @@ -172,7 +207,22 @@ lib_get_event (lnet_eq_t *eq, lnet_event_t *ev) RETURN(rc); } - +/** + * A nonblocking function that can be used to get the next event in an EQ. + * If an event handler is associated with the EQ, the handler will run before + * this function returns successfully. The event is removed from the queue. + * + * \param eventq A handle for the event queue. + * \param event On successful return (1 or -EOVERFLOW), this location will + * hold the next event in the EQ. + * + * \retval 0 No pending event in the EQ. + * \retval 1 Indicates success. + * \retval -ENOENT If \a eventq does not point to a valid EQ. + * \retval -EOVERFLOW Indicates success (i.e., an event is returned) and that + * at least one event between this event and the last event obtained from the + * EQ has been dropped due to limited space in the EQ. + */ int LNetEQGet (lnet_handle_eq_t eventq, lnet_event_t *event) { @@ -182,6 +232,22 @@ LNetEQGet (lnet_handle_eq_t eventq, lnet_event_t *event) event, &which); } +/** + * Block the calling process until there is an event in the EQ. + * If an event handler is associated with the EQ, the handler will run before + * this function returns successfully. This function returns the next event + * in the EQ and removes it from the EQ. + * + * \param eventq A handle for the event queue. + * \param event On successful return (1 or -EOVERFLOW), this location will + * hold the next event in the EQ. + * + * \retval 1 Indicates success. + * \retval -ENOENT If \a eventq does not point to a valid EQ. + * \retval -EOVERFLOW Indicates success (i.e., an event is returned) and that + * at least one event between this event and the last event obtained from the + * EQ has been dropped due to limited space in the EQ. + */ int LNetEQWait (lnet_handle_eq_t eventq, lnet_event_t *event) { @@ -191,6 +257,32 @@ LNetEQWait (lnet_handle_eq_t eventq, lnet_event_t *event) event, &which); } +/** + * Block the calling process until there's an event from a set of EQs or + * timeout happens. + * + * If an event handler is associated with the EQ, the handler will run before + * this function returns successfully, in which case the corresponding event + * is consumed. + * + * LNetEQPoll() provides a timeout to allow applications to poll, block for a + * fixed period, or block indefinitely. + * + * \param eventqs,neq An array of EQ handles, and size of the array. + * \param timeout_ms Time in milliseconds to wait for an event to occur on + * one of the EQs. The constant LNET_TIME_FOREVER can be used to indicate an + * infinite timeout. + * \param event,which On successful return (1 or -EOVERFLOW), \a event will + * hold the next event in the EQs, and \a which will contain the index of the + * EQ from which the event was taken. + * + * \retval 0 No pending event in the EQs after timeout. + * \retval 1 Indicates success. + * \retval -EOVERFLOW Indicates success (i.e., an event is returned) and that + * at least one event between this event and the last event obtained from the + * EQ indicated by \a which has been dropped due to limited space in the EQ. + * \retval -ENOENT If there's an invalid handle in \a eventqs. + */ int LNetEQPoll (lnet_handle_eq_t *eventqs, int neq, int timeout_ms, lnet_event_t *event, int *which) diff --git a/lnet/lnet/lib-md.c b/lnet/lnet/lib-md.c index 7cfce40..c01f2b0 100644 --- a/lnet/lnet/lib-md.c +++ b/lnet/lnet/lib-md.c @@ -230,6 +230,31 @@ lnet_md_validate(lnet_md_t *umd) return 0; } +/** + * Create a memory descriptor and attach it to a ME + * + * \param meh A handle for a ME to associate the new MD with. + * \param umd Provides initial values for the user-visible parts of a MD. + * Other than its use for initialization, there is no linkage between this + * structure and the MD maintained by the LNet. + * \param unlink A flag to indicate whether the MD is automatically unlinked + * when it becomes inactive, either because the operation threshold drops to + * zero or because the available memory becomes less than \a umd.max_size. + * (Note that the check for unlinking a MD only occurs after the completion + * of a successful operation on the MD.) The value LNET_UNLINK enables auto + * unlinking; the value LNET_RETAIN disables it. + * \param handle On successful returns, a handle to the newly created MD is + * saved here. This handle can be used later in LNetMDUnlink(). + * + * \retval 0 On success. + * \retval -EINVAL If \a umd is not valid. + * \retval -ENOMEM If new MD cannot be allocated. + * \retval -ENOENT Either \a meh or \a umd.eq_handle does not point to a + * valid object. Note that it's OK to supply a NULL \a umd.eq_handle by + * calling LNetInvalidateHandle() on it. + * \retval -EBUSY If the ME pointed to by \a meh is already associated with + * a MD. + */ int LNetMDAttach(lnet_handle_me_t meh, lnet_md_t umd, lnet_unlink_t unlink, lnet_handle_md_t *handle) @@ -284,6 +309,22 @@ LNetMDAttach(lnet_handle_me_t meh, lnet_md_t umd, return (rc); } +/** + * Create a "free floating" memory descriptor - a MD that is not associated + * with a ME. Such MDs are usually used in LNetPut() and LNetGet() operations. + * + * \param umd,unlink See the discussion for LNetMDAttach(). + * \param handle On successful returns, a handle to the newly created MD is + * saved here. This handle can be used later in LNetMDUnlink(), LNetPut(), + * and LNetGet() operations. + * + * \retval 0 On success. + * \retval -EINVAL If \a umd is not valid. + * \retval -ENOMEM If new MD cannot be allocated. + * \retval -ENOENT \a umd.eq_handle does not point to a valid EQ. Note that + * it's OK to supply a NULL \a umd.eq_handle by calling + * LNetInvalidateHandle() on it. + */ int LNetMDBind(lnet_md_t umd, lnet_unlink_t unlink, lnet_handle_md_t *handle) { @@ -322,6 +363,35 @@ LNetMDBind(lnet_md_t umd, lnet_unlink_t unlink, lnet_handle_md_t *handle) return (rc); } +/** + * Unlink the memory descriptor from any ME it may be linked to and release + * the internal resources associated with it. + * + * This function does not free the memory region associated with the MD; + * i.e., the memory the user allocated for this MD. If the ME associated with + * this MD is not NULL and was created with auto unlink enabled, the ME is + * unlinked as well (see LNetMEAttach()). + * + * Explicitly unlinking a MD via this function call has the same behavior as + * a MD that has been automatically unlinked, except that no LNET_EVENT_UNLINK + * is generated in the latter case. + * + * An unlinked event can be reported in two ways: + * - If there's no pending operations on the MD, it's unlinked immediately + * and an LNET_EVENT_UNLINK event is logged before this function returns. + * - Otherwise, the MD is only marked for deletion when this function + * returns, and the unlinked event will be piggybacked on the event of + * the completion of the last operation by setting the unlinked field of + * the event. No dedicated LNET_EVENT_UNLINK event is generated. + * + * Note that in both cases the unlinked field of the event is always set; no + * more event will happen on the MD after such an event is logged. + * + * \param mdh A handle for the MD to be unlinked. + * + * \retval 0 On success. + * \retval -ENOENT If \a mdh does not point to a valid MD object. + */ int LNetMDUnlink (lnet_handle_md_t mdh) { diff --git a/lnet/lnet/lib-me.c b/lnet/lnet/lib-me.c index 381e512..d62c970 100644 --- a/lnet/lnet/lib-me.c +++ b/lnet/lnet/lib-me.c @@ -96,6 +96,36 @@ lnet_me_match_portal(lnet_portal_t *ptl, lnet_process_id_t id, return 0; } +/** + * Create and attach a match entry to the match list of \a portal. The new + * ME is empty, i.e. not associated with a memory descriptor. LNetMDAttach() + * can be used to attach a MD to an empty ME. + * + * \param portal The portal table index where the ME should be attached. + * \param match_id Specifies the match criteria for the process ID of + * the requester. The constants LNET_PID_ANY and LNET_NID_ANY can be + * used to wildcard either of the identifiers in the lnet_process_id_t + * structure. + * \param match_bits,ignore_bits Specify the match criteria to apply + * to the match bits in the incoming request. The ignore bits are used + * to mask out insignificant bits in the incoming match bits. The resulting + * bits are then compared to the ME's match bits to determine if the + * incoming request meets the match criteria. + * \param unlink Indicates whether the ME should be unlinked when the memory + * descriptor associated with it is unlinked (Note that the check for + * unlinking a ME only occurs when the memory descriptor is unlinked.). + * Valid values are LNET_RETAIN and LNET_UNLINK. + * \param pos Indicates whether the new ME should be prepended or + * appended to the match list. Allowed constants: LNET_INS_BEFORE, + * LNET_INS_AFTER. + * \param handle On successful returns, a handle to the newly created ME + * object is saved here. This handle can be used later in LNetMEInsert(), + * LNetMEUnlink(), or LNetMDAttach() functions. + * + * \retval 0 On success. + * \retval -EINVAL If \a portal is invalid. + * \retval -ENOMEM If new ME object cannot be allocated. + */ int LNetMEAttach(unsigned int portal, lnet_process_id_t match_id, @@ -148,6 +178,23 @@ LNetMEAttach(unsigned int portal, return 0; } +/** + * Create and a match entry and insert it before or after the ME pointed to by + * \a current_meh. The new ME is empty, i.e. not associated with a memory + * descriptor. LNetMDAttach() can be used to attach a MD to an empty ME. + * + * This function is identical to LNetMEAttach() except for the position + * where the new ME is inserted. + * + * \param current_meh A handle for a ME. The new ME will be inserted + * immediately before or immediately after this ME. + * \param match_id,match_bits,ignore_bits,unlink,pos,handle See the discussion + * for LNetMEAttach(). + * + * \retval 0 On success. + * \retval -ENOMEM If new ME object cannot be allocated. + * \retval -ENOENT If \a current_meh does not point to a valid match entry. + */ int LNetMEInsert(lnet_handle_me_t current_meh, lnet_process_id_t match_id, @@ -207,6 +254,20 @@ LNetMEInsert(lnet_handle_me_t current_meh, return 0; } +/** + * Unlink a match entry from its match list. + * + * This operation also releases any resources associated with the ME. If a + * memory descriptor is attached to the ME, then it will be unlinked as well + * and an unlink event will be generated. It is an error to use the ME handle + * after calling LNetMEUnlink(). + * + * \param meh A handle for the ME to be unlinked. + * + * \retval 0 On success. + * \retval -ENOENT If \a meh does not point to a valid ME. + * \see LNetMDUnlink() for the discussion on delivering unlink event. + */ int LNetMEUnlink(lnet_handle_me_t meh) { diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 957f958..3b87dc3 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -1567,6 +1567,34 @@ lnet_drop_delayed_put(lnet_msg_t *msg, char *reason) LNET_UNLOCK(); } +/** + * Turn on the lazy portal attribute. Use with caution! + * + * This portal attribute only affects incoming PUT requests to the portal, + * and is off by default. By default, if there's no matching MD for an + * incoming PUT request, it is simply dropped. With the lazy attribute on, + * such requests are queued indefinitely until either a matching MD is + * posted to the portal or the lazy attribute is turned off. + * + * It would prevent dropped requests, however it should be regarded as the + * last line of defense - i.e. users must keep a close watch on active + * buffers on a lazy portal and once it becomes too low post more buffers as + * soon as possible. This is because delayed requests usually have detrimental + * effects on underlying network connections. A few delayed requests often + * suffice to bring an underlying connection to a complete halt, due to flow + * control mechanisms. + * + * There's also a DOS attack risk. If users don't post match-all MDs on a + * lazy portal, a malicious peer can easily stop a service by sending some + * PUT requests with match bits that won't match any MD. A routed server is + * especially vulnerable since the connections to its neighbor routers are + * shared among all clients. + * + * \param portal Index of the portal to enable the lazy attribute on. + * + * \retval 0 On success. + * \retval -EINVAL If \a portal is not a valid index. + */ int LNetSetLazyPortal(int portal) { @@ -1584,6 +1612,15 @@ LNetSetLazyPortal(int portal) return 0; } +/** + * Turn off the lazy portal attribute. Delayed requests on the portal, + * if any, will be all dropped when this function returns. + * + * \param portal Index of the portal to disable the lazy attribute on. + * + * \retval 0 On success. + * \retval -EINVAL If \a portal is not a valid index. + */ int LNetClearLazyPortal(int portal) { @@ -2361,6 +2398,50 @@ lnet_parse(lnet_ni_t *ni, lnet_hdr_t *hdr, lnet_nid_t from_nid, return 0; } +/** + * Initiate an asynchronous PUT operation. + * + * There are several events associated with a PUT: completion of the send on + * the initiator node (LNET_EVENT_SEND), and when the send completes + * successfully, the receipt of an acknowledgment (LNET_EVENT_ACK) indicating + * that the operation was accepted by the target. The event LNET_EVENT_PUT is + * used at the target node to indicate the completion of incoming data + * delivery. + * + * The local events will be logged in the EQ associated with the MD pointed to + * by \a mdh handle. Using a MD without an associated EQ results in these + * events being discarded. In this case, the caller must have another + * mechanism (e.g., a higher level protocol) for determining when it is safe + * to modify the memory region associated with the MD. + * + * Note that LNet does not guarantee the order of LNET_EVENT_SEND and + * LNET_EVENT_ACK, though intuitively ACK should happen after SEND. + * + * \param self Indicates the NID of a local interface through which to send + * the PUT request. Use LNET_NID_ANY to let LNet choose one by itself. + * \param mdh A handle for the MD that describes the memory to be sent. The MD + * must be "free floating" (See LNetMDBind()). + * \param ack Controls whether an acknowledgment is requested. + * Acknowledgments are only sent when they are requested by the initiating + * process and the target MD enables them. + * \param target A process identifier for the target process. + * \param portal The index in the \a target's portal table. + * \param match_bits The match bits to use for MD selection at the target + * process. + * \param offset The offset into the target MD (only used when the target + * MD has the LNET_MD_MANAGE_REMOTE option set). + * \param hdr_data 64 bits of user data that can be included in the message + * header. This data is written to an event queue entry at the target if an + * EQ is present on the matching MD. + * + * \retval 0 Success, and only in this case events will be generated + * and logged to EQ (if it exists). + * \retval -EIO Simulated failure. + * \retval -ENOMEM Memory allocation failure. + * \retval -ENOENT Invalid MD object. + * + * \see lnet_event_t::hdr_data and lnet_event_kind_t. + */ int LNetPut(lnet_nid_t self, lnet_handle_md_t mdh, lnet_ack_req_t ack, lnet_process_id_t target, unsigned int portal, @@ -2548,6 +2629,26 @@ lnet_set_reply_msg_len(lnet_ni_t *ni, lnet_msg_t *reply, unsigned int len) reply->msg_ev.mlength = len; } +/** + * Initiate an asynchronous GET operation. + * + * On the initiator node, an LNET_EVENT_SEND is logged when the GET request + * is sent, and an LNET_EVENT_REPLY is logged when the data returned from + * the target node in the REPLY has been written to local MD. + * + * On the target node, an LNET_EVENT_GET is logged when the GET request + * arrives and is accepted into a MD. + * + * \param self,target,portal,match_bits,offset See the discussion in LNetPut(). + * \param mdh A handle for the MD that describes the memory into which the + * requested data will be received. The MD must be "free floating" (See LNetMDBind()). + * + * \retval 0 Success, and only in this case events will be generated + * and logged to EQ (if it exists) of the MD. + * \retval -EIO Simulated failure. + * \retval -ENOMEM Memory allocation failure. + * \retval -ENOENT Invalid MD object. + */ int LNetGet(lnet_nid_t self, lnet_handle_md_t mdh, lnet_process_id_t target, unsigned int portal, @@ -2639,6 +2740,20 @@ LNetGet(lnet_nid_t self, lnet_handle_md_t mdh, return 0; } +/** + * Calculate distance to node at \a dstnid. + * + * \param dstnid Target NID. + * \param srcnidp If not NULL, NID of the local interface to reach \a dstnid + * is saved here. + * \param orderp If not NULL, order of the route to reach \a dstnid is saved + * here. + * + * \retval 0 If \a dstnid belongs to a local interface, and reserved option + * local_nid_dist_zero is set, which is the default. + * \retval positives Distance to target NID, i.e. number of hops plus one. + * \retval -EHOSTUNREACH If \a dstnid is not reachable. + */ int LNetDist (lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp) { @@ -2720,6 +2835,23 @@ LNetDist (lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp) return -EHOSTUNREACH; } +/** + * Set the number of asynchronous messages expected from a target process. + * + * This function is only meaningful for userspace callers. It's a no-op when + * called from kernel. + * + * Asynchronous messages are those that can come from a target when the + * userspace process is not waiting for IO to complete; e.g., AST callbacks + * from Lustre servers. Specifying the expected number of such messages + * allows them to be eagerly received when user process is not running in + * LNet; otherwise network errors may occur. + * + * \param id Process ID of the target process. + * \param nasync Number of asynchronous messages expected from the target. + * + * \return 0 on success, and an error code otherwise. + */ int LNetSetAsync(lnet_process_id_t id, int nasync) { @@ -2737,7 +2869,6 @@ LNetSetAsync(lnet_process_id_t id, int nasync) int rc2; /* Target on a local network? */ - ni = lnet_net2ni(LNET_NIDNET(id.nid)); if (ni != NULL) { if (ni->ni_lnd->lnd_setasync != NULL) diff --git a/lnet/lnet/lib-msg.c b/lnet/lnet/lib-msg.c index 3383d91..4ab5fa4 100644 --- a/lnet/lnet/lib-msg.c +++ b/lnet/lnet/lib-msg.c @@ -72,7 +72,7 @@ lnet_enq_event_locked (lnet_eq_t *eq, lnet_event_t *ev) /* There is no race since both event consumers and event producers * take the LNET_LOCK, so we don't screw around with memory - * barriers, setting the sequence number last or wierd structure + * barriers, setting the sequence number last or weird structure * layout assertions. */ *eq_slot = *ev; -- 1.8.3.1