b=20973 Doxygen comments for LNet API

Doxygen comments for LNet API. This is a comment-only change.

i=maxim

diff --git a/build/doxyfile.api b/build/doxyfile.api
index e871875..9df11fc 100644 (file)
--- a/build/doxyfile.api
+++ b/build/doxyfile.api
@@ -103,9 +103,16 @@ INPUT_ENCODING         = UTF-8
 FILE_PATTERNS          = *.h \
                          *.c
 RECURSIVE              = YES
 FILE_PATTERNS          = *.h \
                          *.c
 RECURSIVE              = YES

-EXCLUDE                =
+EXCLUDE                = lnet/klnds/ciblnd/ \
+                        lnet/klnds/gmlnd/ \
+                        lnet/klnds/iiblnd/ \
+                        lnet/klnds/mxlnd/ \
+                        lnet/klnds/openiblnd/ \
+                        lnet/klnds/qswlnd/ \
+                        lnet/klnds/ralnd/ \
+                        lnet/klnds/viblnd/

 EXCLUDE_SYMLINKS       = NO
 EXCLUDE_SYMLINKS       = NO

-EXCLUDE_PATTERNS       =
+EXCLUDE_PATTERNS       = */darwin/*

 EXCLUDE_SYMBOLS        = EXPORT_SYMBOL*
 EXAMPLE_PATH           =
 EXAMPLE_PATTERNS       = *
 EXCLUDE_SYMBOLS        = EXPORT_SYMBOL*
 EXAMPLE_PATH           =
 EXAMPLE_PATTERNS       = *

diff --git a/build/doxyfile.ref b/build/doxyfile.ref
index ab1445f..4b0602a 100644 (file)
--- a/build/doxyfile.ref
+++ b/build/doxyfile.ref
@@ -103,9 +103,16 @@ INPUT_ENCODING         = UTF-8
 FILE_PATTERNS          = *.h \
                          *.c
 RECURSIVE              = YES
 FILE_PATTERNS          = *.h \
                          *.c
 RECURSIVE              = YES

-EXCLUDE                =
+EXCLUDE                = lnet/klnds/ciblnd/ \
+                        lnet/klnds/gmlnd/ \
+                        lnet/klnds/iiblnd/ \
+                        lnet/klnds/mxlnd/ \
+                        lnet/klnds/openiblnd/ \
+                        lnet/klnds/qswlnd/ \
+                        lnet/klnds/ralnd/ \
+                        lnet/klnds/viblnd/

 EXCLUDE_SYMLINKS       = NO
 EXCLUDE_SYMLINKS       = NO

-EXCLUDE_PATTERNS       =
+EXCLUDE_PATTERNS       = */darwin/*

 EXCLUDE_SYMBOLS        = EXPORT_SYMBOL*
 EXAMPLE_PATH           =
 EXAMPLE_PATTERNS       = *
 EXCLUDE_SYMBOLS        = EXPORT_SYMBOL*
 EXAMPLE_PATH           =
 EXAMPLE_PATTERNS       = *

diff --git a/libcfs/include/libcfs/libcfs_private.h b/libcfs/include/libcfs/libcfs_private.h
index a859e88..51dc986 100644 (file)
--- a/libcfs/include/libcfs/libcfs_private.h
+++ b/libcfs/include/libcfs/libcfs_private.h
@@ -324,14 +324,20 @@ void        cfs_free_nidlist(cfs_list_t *list);
 int         cfs_parse_nidlist(char *str, int len, cfs_list_t *list);
 int         cfs_match_nid(lnet_nid_t nid, cfs_list_t *list);
 
 int         cfs_parse_nidlist(char *str, int len, cfs_list_t *list);
 int         cfs_match_nid(lnet_nid_t nid, cfs_list_t *list);
 

+/** \addtogroup lnet_addr
+ * @{ */

 /* how an LNET NID encodes net:address */
 /* how an LNET NID encodes net:address */

+/** extract the address part of an lnet_nid_t */

 #define LNET_NIDADDR(nid)      ((__u32)((nid) & 0xffffffff))
 #define LNET_NIDADDR(nid)      ((__u32)((nid) & 0xffffffff))

+/** extract the network part of an lnet_nid_t */

 #define LNET_NIDNET(nid)       ((__u32)(((nid) >> 32)) & 0xffffffff)
 #define LNET_NIDNET(nid)       ((__u32)(((nid) >> 32)) & 0xffffffff)

+/** make an lnet_nid_t from a network part and an address part */

 #define LNET_MKNID(net,addr)   ((((__u64)(net))<<32)|((__u64)(addr)))
 /* how net encodes type:number */
 #define LNET_NETNUM(net)       ((net) & 0xffff)
 #define LNET_NETTYP(net)       (((net) >> 16) & 0xffff)
 #define LNET_MKNET(typ,num)    ((((__u32)(typ))<<16)|((__u32)(num)))
 #define LNET_MKNID(net,addr)   ((((__u64)(net))<<32)|((__u64)(addr)))
 /* how net encodes type:number */
 #define LNET_NETNUM(net)       ((net) & 0xffff)
 #define LNET_NETTYP(net)       (((net) >> 16) & 0xffff)
 #define LNET_MKNET(typ,num)    ((((__u32)(typ))<<16)|((__u32)(num)))

+/** @} lnet_addr */

 
 /* max value for numeric network address */
 #define MAX_NUMERIC_VALUE 0xffffffff
 
 /* max value for numeric network address */
 #define MAX_NUMERIC_VALUE 0xffffffff

diff --git a/libcfs/libcfs/watchdog.c b/libcfs/libcfs/watchdog.c
index 274c59b..423a4bd 100644 (file)
--- a/libcfs/libcfs/watchdog.c
+++ b/libcfs/libcfs/watchdog.c
@@ -83,7 +83,7 @@ static unsigned long lcw_flags = 0;
 /*
  * Number of outstanding watchdogs.
  * When it hits 1, we start the dispatcher.
 /*
  * Number of outstanding watchdogs.
  * When it hits 1, we start the dispatcher.

- * When it hits 0, we stop the distpatcher.
+ * When it hits 0, we stop the dispatcher.

  */
 static __u32         lcw_refcount = 0;
 static CFS_DECLARE_MUTEX(lcw_refcount_sem);
  */
 static __u32         lcw_refcount = 0;
 static CFS_DECLARE_MUTEX(lcw_refcount_sem);

diff --git a/lnet/include/lnet/api.h b/lnet/include/lnet/api.h
index 0d5c7f0..9d1c49c 100644 (file)
--- a/lnet/include/lnet/api.h
+++ b/lnet/include/lnet/api.h
@@ -37,28 +37,68 @@
 #ifndef __LNET_API_H__
 #define __LNET_API_H__
 
 #ifndef __LNET_API_H__
 #define __LNET_API_H__
 

+/** \defgroup lnet LNet
+ *
+ * The Lustre Networking subsystem.
+ *
+ * LNet is an asynchronous message-passing API, which provides an unreliable
+ * connectionless service that can't guarantee any order. It supports OFA IB,
+ * TCP/IP, and Cray Portals, and routes between heterogeneous networks.
+ *
+ * LNet can run both in OS kernel space and in userspace as a library.
+ * @{
+ */
+

 #include <lnet/types.h>
 
 #include <lnet/types.h>
 

+/** \defgroup lnet_init_fini Initialization and cleanup
+ * The LNet must be properly initialized before any LNet calls can be made.
+ * @{ */

 int LNetInit(void);
 void LNetFini(void);
 
 int LNetNIInit(lnet_pid_t requested_pid);
 int LNetNIFini(void);
 int LNetInit(void);
 void LNetFini(void);
 
 int LNetNIInit(lnet_pid_t requested_pid);
 int LNetNIFini(void);

+/** @} lnet_init_fini */

 
 

+/** \defgroup lnet_addr LNet addressing and basic types
+ *
+ * Addressing scheme and basic data types of LNet.
+ *
+ * The LNet API is memory-oriented, so LNet must be able to address not only
+ * end-points but also memory region within a process address space.
+ * An ::lnet_nid_t addresses an end-point. An ::lnet_pid_t identifies a process
+ * in a node. A portal represents an opening in the address space of a
+ * process. Match bits is criteria to identify a region of memory inside a
+ * portal, and offset specifies an offset within the memory region.
+ *
+ * LNet creates a table of portals for each process during initialization.
+ * This table has MAX_PORTALS entries and its size can't be dynamically
+ * changed. A portal stays empty until the owning process starts to add
+ * memory regions to it. A portal is sometimes called an index because
+ * it's an entry in the portals table of a process.
+ *
+ * \see LNetMEAttach
+ * @{ */

 int LNetGetId(unsigned int index, lnet_process_id_t *id);
 int LNetDist(lnet_nid_t nid, lnet_nid_t *srcnid, __u32 *order);
 int LNetGetId(unsigned int index, lnet_process_id_t *id);
 int LNetDist(lnet_nid_t nid, lnet_nid_t *srcnid, __u32 *order);

-int LNetCtl(unsigned int cmd, void *arg);
-void LNetSnprintHandle (char *str, int str_len, lnet_handle_any_t handle);
+void LNetSnprintHandle(char *str, int str_len, lnet_handle_any_t handle);

 
 

-/*
- * Portals
- */
-int LNetSetLazyPortal(int portal);
-int LNetClearLazyPortal(int portal);
+/** @} lnet_addr */

 
 

-/*
- * Match entries
- */
+
+/** \defgroup lnet_me Match entries
+ *
+ * A match entry (abbreviated as ME) describes a set of criteria to accept
+ * incoming requests.
+ *
+ * A portal is essentially a match list plus a set of attributes. A match
+ * list is a chain of MEs. Each ME includes a pointer to a memory descriptor
+ * and a set of match criteria. The match criteria can be used to reject
+ * incoming requests based on process ID or the match bits provided in the
+ * request. MEs can be dynamically inserted into a match list by LNetMEAttach()
+ * and LNetMEInsert(), and removed from its list by LNetMEUnlink().
+ * @{ */

 int LNetMEAttach(unsigned int      portal,
                 lnet_process_id_t match_id_in, 
                 __u64             match_bits_in,
 int LNetMEAttach(unsigned int      portal,
                 lnet_process_id_t match_id_in, 
                 __u64             match_bits_in,


@@ -76,10 +116,20 @@ int LNetMEInsert(lnet_handle_me_t  current_in,
                 lnet_handle_me_t *handle_out);
 
 int LNetMEUnlink(lnet_handle_me_t current_in);
                 lnet_handle_me_t *handle_out);
 
 int LNetMEUnlink(lnet_handle_me_t current_in);

+/** @} lnet_me */

 
 

-/*
- * Memory descriptors
- */
+/** \defgroup lnet_md Memory descriptors
+ *
+ * A memory descriptor contains information about a region of a user's
+ * memory (either in kernel or user space) and optionally points to an
+ * event queue where information about the operations performed on the
+ * memory descriptor are recorded. Memory descriptor is abbreviated as
+ * MD and can be used interchangeably with the memory region it describes.
+ *
+ * The LNet API provides two operations to create MDs: LNetMDAttach()
+ * and LNetMDBind(); one operation to unlink and release the resources
+ * associated with a MD: LNetMDUnlink().
+ * @{ */

 int LNetMDAttach(lnet_handle_me_t  current_in, 
                 lnet_md_t         md_in,
                 lnet_unlink_t     unlink_in, 
 int LNetMDAttach(lnet_handle_me_t  current_in, 
                 lnet_md_t         md_in,
                 lnet_unlink_t     unlink_in, 


@@ -90,10 +140,30 @@ int LNetMDBind(lnet_md_t         md_in,
               lnet_handle_md_t *handle_out);
 
 int LNetMDUnlink(lnet_handle_md_t md_in);
               lnet_handle_md_t *handle_out);
 
 int LNetMDUnlink(lnet_handle_md_t md_in);

+/** @} lnet_md */

 
 

-/*
- * Event queues
- */
+/** \defgroup lnet_eq Events and event queues
+ *
+ * Event queues (abbreviated as EQ) are used to log operations performed on
+ * local MDs. In particular, they signal the completion of a data transmission
+ * into or out of a MD. They can also be used to hold acknowledgments for
+ * completed PUT operations and indicate when a MD has been unlinked. Multiple
+ * MDs can share a single EQ. An EQ may have an optional event handler
+ * associated with it. If an event handler exists, it will be run for each
+ * event that is deposited into the EQ.
+ *
+ * In addition to the lnet_handle_eq_t, the LNet API defines two types
+ * associated with events: The ::lnet_event_kind_t defines the kinds of events
+ * that can be stored in an EQ. The lnet_event_t defines a structure that
+ * holds the information about with an event.
+ *
+ * There are five functions for dealing with EQs: LNetEQAlloc() is used to
+ * create an EQ and allocate the resources needed, while LNetEQFree()
+ * releases these resources and free the EQ. LNetEQGet() retrieves the next
+ * event from an EQ, and LNetEQWait() can be used to block a process until
+ * an EQ has at least one event. LNetEQPoll() can be used to test or wait
+ * on multiple EQs.
+ * @{ */

 int LNetEQAlloc(unsigned int       count_in,
                lnet_eq_handler_t  handler,
                lnet_handle_eq_t  *handle_out);
 int LNetEQAlloc(unsigned int       count_in,
                lnet_eq_handler_t  handler,
                lnet_handle_eq_t  *handle_out);


@@ -112,10 +182,13 @@ int LNetEQPoll(lnet_handle_eq_t *eventqs_in,
               int               timeout_ms,
               lnet_event_t     *event_out, 
               int              *which_eq_out);
               int               timeout_ms,
               lnet_event_t     *event_out, 
               int              *which_eq_out);

+/** @} lnet_eq */

 
 

-/*
- * Data movement
- */
+/** \defgroup lnet_data Data movement operations
+ *
+ * The LNet API provides two data movement operations: LNetPut()
+ * and LNetGet().
+ * @{ */

 int LNetPut(lnet_nid_t        self,
            lnet_handle_md_t  md_in, 
            lnet_ack_req_t    ack_req_in,
 int LNetPut(lnet_nid_t        self,
            lnet_handle_md_t  md_in, 
            lnet_ack_req_t    ack_req_in,


@@ -131,8 +204,16 @@ int LNetGet(lnet_nid_t        self,
            unsigned int      portal_in, 
            __u64             match_bits_in, 
            unsigned int      offset_in);
            unsigned int      portal_in, 
            __u64             match_bits_in, 
            unsigned int      offset_in);

+/** @} lnet_data */
+

 
 

+/** \defgroup lnet_misc Miscellaneous operations.
+ * Miscellaneous operations.
+ * @{ */

 
 

+int LNetSetLazyPortal(int portal);
+int LNetClearLazyPortal(int portal);
+int LNetCtl(unsigned int cmd, void *arg);

 int LNetSetAsync(lnet_process_id_t id, int nasync);
 
 #ifndef __KERNEL__
 int LNetSetAsync(lnet_process_id_t id, int nasync);
 
 #ifndef __KERNEL__


@@ -141,6 +222,8 @@ int LNetSetAsync(lnet_process_id_t id, int nasync);
  * lnet/lib-types.h */
 
 void lnet_server_mode();
  * lnet/lib-types.h */
 
 void lnet_server_mode();

-#endif        
+#endif
+/** @} lnet_misc */

 
 

+/** @} lnet */

 #endif
 #endif

diff --git a/lnet/include/lnet/types.h b/lnet/include/lnet/types.h
index e971548..bb6a5f0 100644 (file)
--- a/lnet/include/lnet/types.h
+++ b/lnet/include/lnet/types.h
@@ -37,14 +37,39 @@
 #ifndef __LNET_TYPES_H__
 #define __LNET_TYPES_H__
 
 #ifndef __LNET_TYPES_H__
 #define __LNET_TYPES_H__
 

+/** \addtogroup lnet
+ * @{ */
+

 #include <libcfs/libcfs.h>
 
 #include <libcfs/libcfs.h>
 

-#define LNET_RESERVED_PORTAL      0  /* portals reserved for lnet's own use */
+/** \addtogroup lnet_addr
+ * @{ */
+
+/** Portal reserved for LNet's own use.
+ * \see lustre/include/lustre/lustre_idl.h for Lustre portal assignments.
+ */
+#define LNET_RESERVED_PORTAL      0

 
 

+/**
+ * Address of an end-point in an LNet network.
+ *
+ * A node can have multiple end-points and hence multiple addresses.
+ * An LNet network can be a simple network (e.g. tcp0) or a network of
+ * LNet networks connected by LNet routers. Therefore an end-point address
+ * has two parts: network ID, and address within a network.
+ *
+ * \see LNET_NIDNET, LNET_NIDADDR, and LNET_MKNID.
+ */

 typedef __u64 lnet_nid_t;
 typedef __u64 lnet_nid_t;

+/**
+ * ID of a process in a node. Shortened as PID to distinguish from
+ * lnet_process_id_t, the global process ID.
+ */

 typedef __u32 lnet_pid_t;
 
 typedef __u32 lnet_pid_t;
 

+/** wildcard NID that matches any end-point address */

 #define LNET_NID_ANY      ((lnet_nid_t) -1)
 #define LNET_NID_ANY      ((lnet_nid_t) -1)

+/** wildcard PID that matches any lnet_pid_t */

 #define LNET_PID_ANY      ((lnet_pid_t) -1)
 
 #ifdef CRAY_XT3
 #define LNET_PID_ANY      ((lnet_pid_t) -1)
 
 #ifdef CRAY_XT3


@@ -57,6 +82,16 @@ typedef __u32 lnet_uid_t;
 
 #define LNET_TIME_FOREVER    (-1)
 
 
 #define LNET_TIME_FOREVER    (-1)
 

+/**
+ * Objects maintained by the LNet are accessed through handles. Handle types
+ * have names of the form lnet_handle_xx_t, where xx is one of the two letter
+ * object type codes ('eq' for event queue, 'md' for memory descriptor, and
+ * 'me' for match entry).
+ * Each type of object is given a unique handle type to enhance type checking.
+ * The type lnet_handle_any_t can be used when a generic handle is needed.
+ * Every handle value can be converted into a value of type lnet_handle_any_t
+ * without loss of information.
+ */

 typedef struct {
         __u64         cookie;
 } lnet_handle_any_t;
 typedef struct {
         __u64         cookie;
 } lnet_handle_any_t;


@@ -67,43 +102,171 @@ typedef lnet_handle_any_t lnet_handle_me_t;
 
 #define LNET_WIRE_HANDLE_COOKIE_NONE   (-1)
 
 
 #define LNET_WIRE_HANDLE_COOKIE_NONE   (-1)
 

+/**
+ * Invalidate handle \a h.
+ */

 static inline void LNetInvalidateHandle(lnet_handle_any_t *h)
 {
         h->cookie = LNET_WIRE_HANDLE_COOKIE_NONE;
 }
 
 static inline void LNetInvalidateHandle(lnet_handle_any_t *h)
 {
         h->cookie = LNET_WIRE_HANDLE_COOKIE_NONE;
 }
 

+/**
+ * Compare handles \a h1 and \a h2.
+ *
+ * \return 1 if handles are equal, 0 if otherwise.
+ */

 static inline int LNetHandleIsEqual (lnet_handle_any_t h1, lnet_handle_any_t h2)
 {
         return (h1.cookie == h2.cookie);
 }
 
 static inline int LNetHandleIsEqual (lnet_handle_any_t h1, lnet_handle_any_t h2)
 {
         return (h1.cookie == h2.cookie);
 }
 

+/**
+ * Check whether handle \a h is invalid.
+ *
+ * \return 1 if handle is invalid, 0 if valid.
+ */

 static inline int LNetHandleIsInvalid(lnet_handle_any_t h)
 {
         return (LNET_WIRE_HANDLE_COOKIE_NONE == h.cookie);
 }
 
 static inline int LNetHandleIsInvalid(lnet_handle_any_t h)
 {
         return (LNET_WIRE_HANDLE_COOKIE_NONE == h.cookie);
 }
 

+/**
+ * Global process ID.
+ */

 typedef struct {
 typedef struct {

+        /** node id */

         lnet_nid_t nid;
         lnet_nid_t nid;

-        lnet_pid_t pid;   /* node id / process id */
+        /** process id */
+        lnet_pid_t pid;

 } lnet_process_id_t;
 } lnet_process_id_t;

+/** @} lnet_addr */
+
+/** \addtogroup lnet_me
+ * @{ */

 
 

+/**
+ * Specifies whether the match entry or memory descriptor should be unlinked
+ * automatically (LNET_UNLINK) or not (LNET_RETAIN).
+ */

 typedef enum {
         LNET_RETAIN = 0,
         LNET_UNLINK
 } lnet_unlink_t;
 
 typedef enum {
         LNET_RETAIN = 0,
         LNET_UNLINK
 } lnet_unlink_t;
 

+/**
+ * Values of the type lnet_ins_pos_t are used to control where a new match
+ * entry is inserted. The value LNET_INS_BEFORE is used to insert the new
+ * entry before the current entry or before the head of the list. The value
+ * LNET_INS_AFTER is used to insert the new entry after the current entry
+ * or after the last item in the list.
+ */

 typedef enum {
         LNET_INS_BEFORE,
         LNET_INS_AFTER
 } lnet_ins_pos_t;
 
 typedef enum {
         LNET_INS_BEFORE,
         LNET_INS_AFTER
 } lnet_ins_pos_t;
 

+/** @} lnet_me */
+
+/** \addtogroup lnet_md
+ * @{ */
+
+/**
+ * Defines the visible parts of a memory descriptor. Values of this type
+ * are used to initialize memory descriptors.
+ */

 typedef struct {
 typedef struct {

+        /**
+         * Specify the memory region associated with the memory descriptor.
+         * If the options field has:
+         * - LNET_MD_KIOV bit set: The start field points to the starting
+         * address of an array of lnet_kiov_t and the length field specifies
+         * the number of entries in the array. The length can't be bigger
+         * than LNET_MAX_IOV. The lnet_kiov_t is used to describe page-based
+         * fragments that are not necessarily mapped in virtal memory.
+         * - LNET_MD_IOVEC bit set: The start field points to the starting
+         * address of an array of struct iovec and the length field specifies
+         * the number of entries in the array. The length can't be bigger
+         * than LNET_MAX_IOV. The struct iovec is used to describe fragments
+         * that have virtual addresses.
+         * - Otherwise: The memory region is contiguous. The start field
+         * specifies the starting address for the memory region and the
+         * length field specifies its length.
+         *
+         * When the memory region is fragmented, all fragments but the first
+         * one must start on page boundary, and all but the last must end on
+         * page boundary.
+         */

         void            *start;
         unsigned int     length;
         void            *start;
         unsigned int     length;

+        /**
+         * Specifies the maximum number of operations that can be performed
+         * on the memory descriptor. An operation is any action that could
+         * possibly generate an event. In the usual case, the threshold value
+         * is decremented for each operation on the MD. When the threshold
+         * drops to zero, the MD becomes inactive and does not respond to
+         * operations. A threshold value of LNET_MD_THRESH_INF indicates that
+         * there is no bound on the number of operations that may be applied
+         * to a MD.
+         */

         int              threshold;
         int              threshold;

+        /**
+         * Specifies the largest incoming request that the memory descriptor
+         * should respond to. When the unused portion of a MD (length -
+         * local offset) falls below this value, the MD becomes inactive and
+         * does not respond to further operations. This value is only used
+         * if the LNET_MD_MAX_SIZE option is set.
+         */

         int              max_size;
         int              max_size;

+        /**
+         * Specifies the behavior of the memory descriptor. A bitwise OR
+         * of the following values can be used:
+         * - LNET_MD_OP_PUT: The LNet PUT operation is allowed on this MD.
+         * - LNET_MD_OP_GET: The LNet GET operation is allowed on this MD.
+         * - LNET_MD_MANAGE_REMOTE: The offset used in accessing the memory
+         *   region is provided by the incoming request. By default, the
+         *   offset is maintained locally. When maintained locally, the
+         *   offset is incremented by the length of the request so that
+         *   the next operation (PUT or GET) will access the next part of
+         *   the memory region. Note that only one offset variable exists
+         *   per memory descriptor. If both PUT and GET operations are
+         *   performed on a memory descriptor, the offset is updated each time.
+         * - LNET_MD_TRUNCATE: The length provided in the incoming request can
+         *   be reduced to match the memory available in the region (determined
+         *   by subtracting the offset from the length of the memory region).
+         *   By default, if the length in the incoming operation is greater
+         *   than the amount of memory available, the operation is rejected.
+         * - LNET_MD_ACK_DISABLE: An acknowledgment should not be sent for
+         *   incoming PUT operations, even if requested. By default,
+         *   acknowledgments are sent for PUT operations that request an
+         *   acknowledgment. Acknowledgments are never sent for GET operations.
+         *   The data sent in the REPLY serves as an implicit acknowledgment.
+         * - LNET_MD_KIOV: The start and length fields specify an array of
+         *   lnet_kiov_t.
+         * - LNET_MD_IOVEC: The start and length fields specify an array of
+         *   struct iovec.
+         * - LNET_MD_MAX_SIZE: The max_size field is valid.
+         *
+         * Note:
+         * - LNET_MD_KIOV or LNET_MD_IOVEC allows for a scatter/gather
+         *   capability for memory descriptors. They can't be both set.
+         * - When LNET_MD_MAX_SIZE is set, the total length of the memory
+         *   region (i.e. sum of all fragment lengths) must not be less than
+         *   \a max_size.
+         */

         unsigned int     options;
         unsigned int     options;

+        /**
+         * A user-specified value that is associated with the memory
+         * descriptor. The value does not need to be a pointer, but must fit
+         * in the space used by a pointer. This value is recorded in events
+         * associated with operations on this MD.
+         */

         void            *user_ptr;
         void            *user_ptr;

+        /**
+         * A handle for the event queue used to log the operations performed on
+         * the memory region. If this argument is a NULL handle (i.e. nullified
+         * by LNetInvalidateHandle()), operations performed on this memory
+         * descriptor are not logged.
+         */

         lnet_handle_eq_t eq_handle;
 } lnet_md_t;
 
         lnet_handle_eq_t eq_handle;
 } lnet_md_t;
 


@@ -111,7 +274,7 @@ typedef struct {
 #define LNET_MTU_BITS   20
 #define LNET_MTU        (1<<LNET_MTU_BITS)
 
 #define LNET_MTU_BITS   20
 #define LNET_MTU        (1<<LNET_MTU_BITS)
 

-/* limit on the number of entries in discontiguous MDs */
+/** limit on the number of fragments in discontiguous MDs */

 #define LNET_MAX_IOV    256
 
 /* Max payload size */
 #define LNET_MAX_IOV    256
 
 /* Max payload size */


@@ -128,37 +291,93 @@ typedef struct {
 # endif
 #endif
 
 # endif
 #endif
 

-/* Options for the MD structure */
+/**
+ * Options for the MD structure. See lnet_md_t::options.
+ */

 #define LNET_MD_OP_PUT               (1 << 0)
 #define LNET_MD_OP_PUT               (1 << 0)

+/** See lnet_md_t::options. */

 #define LNET_MD_OP_GET               (1 << 1)
 #define LNET_MD_OP_GET               (1 << 1)

+/** See lnet_md_t::options. */

 #define LNET_MD_MANAGE_REMOTE        (1 << 2)
 /* unused                            (1 << 3) */
 #define LNET_MD_MANAGE_REMOTE        (1 << 2)
 /* unused                            (1 << 3) */

+/** See lnet_md_t::options. */

 #define LNET_MD_TRUNCATE             (1 << 4)
 #define LNET_MD_TRUNCATE             (1 << 4)

+/** See lnet_md_t::options. */

 #define LNET_MD_ACK_DISABLE          (1 << 5)
 #define LNET_MD_ACK_DISABLE          (1 << 5)

+/** See lnet_md_t::options. */

 #define LNET_MD_IOVEC                (1 << 6)
 #define LNET_MD_IOVEC                (1 << 6)

+/** See lnet_md_t::options. */

 #define LNET_MD_MAX_SIZE             (1 << 7)
 #define LNET_MD_MAX_SIZE             (1 << 7)

+/** See lnet_md_t::options. */

 #define LNET_MD_KIOV                 (1 << 8)
 
 /* For compatibility with Cray Portals */
 #define LNET_MD_PHYS                         0
 
 #define LNET_MD_KIOV                 (1 << 8)
 
 /* For compatibility with Cray Portals */
 #define LNET_MD_PHYS                         0
 

+/** Infinite threshold on MD operations. See lnet_md_t::threshold */

 #define LNET_MD_THRESH_INF       (-1)
 
 /* NB lustre portals uses struct iovec internally! */
 typedef struct iovec lnet_md_iovec_t;
 
 #define LNET_MD_THRESH_INF       (-1)
 
 /* NB lustre portals uses struct iovec internally! */
 typedef struct iovec lnet_md_iovec_t;
 

+/**
+ * A page-based fragment of a MD.
+ */

 typedef struct {
 typedef struct {

+        /** Pointer to the page where the fragment resides */

         cfs_page_t      *kiov_page;
         cfs_page_t      *kiov_page;

+        /** Length in bytes of the fragment */

         unsigned int     kiov_len;
         unsigned int     kiov_len;

+        /**
+         * Starting offset of the fragment within the page. Note that the
+         * end of the fragment must not pass the end of the page; i.e.,
+         * kiov_len + kiov_offset <= CFS_PAGE_SIZE.
+         */

         unsigned int     kiov_offset;
 } lnet_kiov_t;
         unsigned int     kiov_offset;
 } lnet_kiov_t;

+/** @} lnet_md */

 
 

+/** \addtogroup lnet_eq
+ * @{ */
+
+/**
+ * Six types of events can be logged in an event queue.
+ */

 typedef enum {
 typedef enum {

+        /** An incoming GET operation has completed on the MD. */

         LNET_EVENT_GET,
         LNET_EVENT_GET,

+        /**
+         * An incoming PUT operation has completed on the MD. The
+         * underlying layers will not alter the memory (on behalf of this
+         * operation) once this event has been logged.
+         */

         LNET_EVENT_PUT,
         LNET_EVENT_PUT,

+        /**
+         * A REPLY operation has completed. This event is logged after the
+         * data (if any) from the REPLY has been written into the MD.
+         */

         LNET_EVENT_REPLY,
         LNET_EVENT_REPLY,

+        /** An acknowledgment has been received. */

         LNET_EVENT_ACK,
         LNET_EVENT_ACK,

+        /**
+         * An outgoing send (PUT or GET) operation has completed. This event
+         * is logged after the entire buffer has been sent and it is safe for
+         * the caller to reuse the buffer.
+         *
+         * Note:
+         * - The LNET_EVENT_SEND doesn't guarantee message delivery. It can
+         *   happen even when the message has not yet been put out on wire.
+         * - It's unsafe to assume that in an outgoing GET operation
+         *   the LNET_EVENT_SEND event would happen before the
+         *   LNET_EVENT_REPLY event. The same holds for LNET_EVENT_SEND and
+         *   LNET_EVENT_ACK events in an outgoing PUT operation.
+         */

         LNET_EVENT_SEND,
         LNET_EVENT_SEND,

+        /**
+         * A MD has been unlinked. Note that LNetMDUnlink() does not
+         * necessarily trigger an LNET_EVENT_UNLINK event.
+         * \see LNetMDUnlink
+         */

         LNET_EVENT_UNLINK,
 } lnet_event_kind_t;
 
         LNET_EVENT_UNLINK,
 } lnet_event_kind_t;
 


@@ -173,37 +392,118 @@ typedef unsigned LNET_SEQ_BASETYPE lnet_seq_t;
 #ifdef __CYGWIN__
 #pragma pack(push, 4)
 #endif
 #ifdef __CYGWIN__
 #pragma pack(push, 4)
 #endif

+
+/**
+ * Information about an event on a MD.
+ */

 typedef struct {
 typedef struct {

+        /** The identifier (nid, pid) of the target. */

         lnet_process_id_t   target;
         lnet_process_id_t   target;

+        /** The identifier (nid, pid) of the initiator. */

         lnet_process_id_t   initiator;
         lnet_process_id_t   initiator;

+        /**
+         * The NID of the immediate sender. If the request has been forwarded
+         * by routers, this is the NID of the last hop; otherwise it's the
+         * same as the initiator.
+         */

         lnet_nid_t          sender;
         lnet_nid_t          sender;

+        /** Indicates the type of the event. */

         lnet_event_kind_t   type;
         lnet_event_kind_t   type;

+        /** The portal table index specified in the request */

         unsigned int        pt_index;
         unsigned int        pt_index;

+        /** A copy of the match bits specified in the request. */

         __u64               match_bits;
         __u64               match_bits;

+        /** The length (in bytes) specified in the request. */

         unsigned int        rlength;
         unsigned int        rlength;

+        /**
+         * The length (in bytes) of the data that was manipulated by the
+         * operation. For truncated operations, the manipulated length will be
+         * the number of bytes specified by the MD (possibly with an offset,
+         * see lnet_md_t). For all other operations, the manipulated length
+         * will be the length of the requested operation, i.e. rlength.
+         */

         unsigned int        mlength;
         unsigned int        mlength;

+        /**
+         * The handle to the MD associated with the event. The handle may be
+         * invalid if the MD has been unlinked.
+         */

         lnet_handle_md_t    md_handle;
         lnet_handle_md_t    md_handle;

+        /**
+         * A snapshot of the state of the MD immediately after the event has
+         * been processed. In particular, the threshold field in md will
+         * reflect the value of the threshold after the operation occurred.
+         */

         lnet_md_t           md;
         lnet_md_t           md;

+        /**
+         * 64 bits of out-of-band user data. Only valid for LNET_EVENT_PUT.
+         * \see LNetPut
+         */

         __u64               hdr_data;
         __u64               hdr_data;

+        /**
+         * Indicates the completion status of the operation. It's 0 for
+         * successful operations, otherwise it's an error code.
+         */

         int                 status;
         int                 status;

+        /**
+         * Indicates whether the MD has been unlinked. Note that:
+         * - An event with unlinked set is the last event on the MD.
+         * - This field is also set for an explicit LNET_EVENT_UNLINK event.
+         * \see LNetMDUnlink
+         */

         int                 unlinked;
         int                 unlinked;

+        /**
+         * The displacement (in bytes) into the memory region that the
+         * operation used. The offset can be determined by the operation for
+         * a remote managed MD or by the local MD.
+         * \see lnet_md_t::options
+         */

         unsigned int        offset;
 #ifdef CRAY_XT3
         lnet_uid_t          uid;
 #endif
 
         unsigned int        offset;
 #ifdef CRAY_XT3
         lnet_uid_t          uid;
 #endif
 

+        /**
+         * The sequence number for this event. Sequence numbers are unique
+         * to each event.
+         */

         volatile lnet_seq_t sequence;
 } lnet_event_t;
 #ifdef __CYGWIN__
 #pragma pop
 #endif
 
         volatile lnet_seq_t sequence;
 } lnet_event_t;
 #ifdef __CYGWIN__
 #pragma pop
 #endif
 

+/**
+ * Event queue handler function type.
+ *
+ * The EQ handler runs for each event that is deposited into the EQ. The
+ * handler is supplied with a pointer to the event that triggered the
+ * handler invocation.
+ *
+ * The handler must not block, must be reentrant, and must not call any LNet
+ * API functions. It should return as quickly as possible.
+ */
+typedef void (*lnet_eq_handler_t)(lnet_event_t *event);
+#define LNET_EQ_HANDLER_NONE NULL
+/** @} lnet_eq */
+
+/** \addtogroup lnet_data
+ * @{ */
+
+/**
+ * Specify whether an acknowledgment should be sent by target when the PUT
+ * operation completes (i.e., when the data has been written to a MD of the
+ * target process).
+ *
+ * \see lnet_md_t::options for the discussion on LNET_MD_ACK_DISABLE by which
+ * acknowledgments can be disabled for a MD.
+ */

 typedef enum {
 typedef enum {

+        /** Request an acknowledgment */

         LNET_ACK_REQ,
         LNET_ACK_REQ,

+        /** Request that no acknowledgment should be generated. */

         LNET_NOACK_REQ
 } lnet_ack_req_t;
         LNET_NOACK_REQ
 } lnet_ack_req_t;

+/** @} lnet_data */

 
 

-typedef void (*lnet_eq_handler_t)(lnet_event_t *event);
-#define LNET_EQ_HANDLER_NONE NULL
-
+/** @} lnet */

 #endif
 #endif

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 0a3fabb..07069db 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -633,7 +633,9 @@ lnet_fini_finalizers(void)
 }
 
 #ifndef __KERNEL__
 }
 
 #ifndef __KERNEL__

-/* Temporary workaround to allow uOSS and test programs force server
+/**
+ * Reserved API - do not use.
+ * Temporary workaround to allow uOSS and test programs force server

  * mode in userspace. See comments near ln_server_mode_flag in
  * lnet/lib-types.h */
 
  * mode in userspace. See comments near ln_server_mode_flag in
  * lnet/lib-types.h */
 


@@ -1152,6 +1154,16 @@ lnet_startup_lndnis (void)
         return -ENETDOWN;
 }
 
         return -ENETDOWN;
 }
 

+/**
+ * Initialize LNet library.
+ *
+ * Only userspace program needs to call this function - it's automatically
+ * called in the kernel at module loading time. Caller has to call LNetFini()
+ * after a call to LNetInit(), if and only if the latter returned 0. It must
+ * be called exactly once.
+ *
+ * \return 0 on success, and -ve on failures.
+ */

 int
 LNetInit(void)
 {
 int
 LNetInit(void)
 {


@@ -1185,6 +1197,15 @@ LNetInit(void)
         return 0;
 }
 
         return 0;
 }
 

+/**
+ * Finalize LNet library.
+ *
+ * Only userspace program needs to call this function. It can be called
+ * at most once.
+ *
+ * \pre LNetInit() called with success.
+ * \pre All LNet users called LNetNIFini() for matching LNetNIInit() calls.
+ */

 void
 LNetFini(void)
 {
 void
 LNetFini(void)
 {


@@ -1199,6 +1220,22 @@ LNetFini(void)
         the_lnet.ln_init = 0;
 }
 
         the_lnet.ln_init = 0;
 }
 

+/**
+ * Set LNet PID and start LNet interfaces, routing, and forwarding.
+ *
+ * Userspace program should call this after a successful call to LNetInit().
+ * Users must call this function at least once before any other functions.
+ * For each successful call there must be a corresponding call to
+ * LNetNIFini(). For subsequent calls to LNetNIInit(), \a requested_pid is
+ * ignored.
+ *
+ * The PID used by LNet may be different from the one requested.
+ * See LNetGetId().
+ *
+ * \param requested_pid PID requested by the caller.
+ *
+ * \return 0 on success, and non-zero error code on failures.
+ */

 int
 LNetNIInit(lnet_pid_t requested_pid)
 {
 int
 LNetNIInit(lnet_pid_t requested_pid)
 {


@@ -1280,6 +1317,15 @@ LNetNIInit(lnet_pid_t requested_pid)
         return rc;
 }
 
         return rc;
 }
 

+/**
+ * Stop LNet interfaces, routing, and forwarding.
+ *
+ * Users must call this function once for each successful call to LNetNIInit().
+ * Once the LNetNIFini() operation has been started, the results of pending
+ * API operations are undefined.
+ *
+ * \return always 0 for current implementation.
+ */

 int
 LNetNIFini()
 {
 int
 LNetNIFini()
 {


@@ -1310,6 +1356,19 @@ LNetNIFini()
         return 0;
 }
 
         return 0;
 }
 

+/**
+ * This is an ugly hack to export IOC_LIBCFS_DEBUG_PEER and
+ * IOC_LIBCFS_PORTALS_COMPATIBILITY commands to users, by tweaking the LNet
+ * internal ioctl handler.
+ *
+ * IOC_LIBCFS_PORTALS_COMPATIBILITY is now deprecated, don't use it.
+ *
+ * \param cmd IOC_LIBCFS_DEBUG_PEER to print debugging data about a peer.
+ * The data will be printed to system console. Don't use it excessively.
+ * \param arg A pointer to lnet_process_id_t, process ID of the peer.
+ *
+ * \return Always return 0 when called by users directly (i.e., not via ioctl).
+ */

 int
 LNetCtl(unsigned int cmd, void *arg)
 {
 int
 LNetCtl(unsigned int cmd, void *arg)
 {


@@ -1416,6 +1475,17 @@ LNetCtl(unsigned int cmd, void *arg)
         /* not reached */
 }
 
         /* not reached */
 }
 

+/**
+ * Retrieve the lnet_process_id_t ID of LNet interface at \a index. Note that
+ * all interfaces share a same PID, as requested by LNetNIInit().
+ *
+ * \param index Index of the interface to look up.
+ * \param id On successful return, this location will hold the
+ * lnet_process_id_t ID of the interface.
+ *
+ * \retval 0 If an interface exists at \a index.
+ * \retval -ENOENT If no interface has been found.
+ */

 int
 LNetGetId(unsigned int index, lnet_process_id_t *id)
 {
 int
 LNetGetId(unsigned int index, lnet_process_id_t *id)
 {


@@ -1445,6 +1515,10 @@ LNetGetId(unsigned int index, lnet_process_id_t *id)
         return rc;
 }
 
         return rc;
 }
 

+/**
+ * Print a string representation of handle \a h into buffer \a str of
+ * \a len bytes.
+ */

 void
 LNetSnprintHandle(char *str, int len, lnet_handle_any_t h)
 {
 void
 LNetSnprintHandle(char *str, int len, lnet_handle_any_t h)
 {

diff --git a/lnet/lnet/lib-eq.c b/lnet/lnet/lib-eq.c
index b4e4ccc..d0991d9 100644 (file)
--- a/lnet/lnet/lib-eq.c
+++ b/lnet/lnet/lib-eq.c
@@ -41,6 +41,31 @@
 #define DEBUG_SUBSYSTEM S_LNET
 #include <lnet/lib-lnet.h>
 
 #define DEBUG_SUBSYSTEM S_LNET
 #include <lnet/lib-lnet.h>
 

+/**
+ * Create an event queue that has room for \a count number of events.
+ *
+ * The event queue is circular and older events will be overwritten by new
+ * ones if they are not removed in time by the user using the functions
+ * LNetEQGet(), LNetEQWait(), or LNetEQPoll(). It is up to the user to
+ * determine the appropriate size of the event queue to prevent this loss
+ * of events. Note that when EQ handler is specified in \a callback, no
+ * event loss can happen, since the handler is run for each event deposited
+ * into the EQ.
+ *
+ * \param count The number of events to be stored in the event queue. It
+ * will be rounded up to the next power of two.
+ * \param callback A handler function that runs when an event is deposited
+ * into the EQ. The constant value LNET_EQ_HANDLER_NONE can be used to
+ * indicate that no event handler is desired.
+ * \param handle On successful return, this location will hold a handle for
+ * the newly created EQ.
+ *
+ * \retval 0       On success.
+ * \retval -EINVAL If an parameter is not valid.
+ * \retval -ENOMEM If memory for the EQ can't be allocated.
+ *
+ * \see lnet_eq_handler_t for the discussion on EQ handler semantics.
+ */

 int
 LNetEQAlloc(unsigned int count, lnet_eq_handler_t callback,
             lnet_handle_eq_t *handle)
 int
 LNetEQAlloc(unsigned int count, lnet_eq_handler_t callback,
             lnet_handle_eq_t *handle)


@@ -52,7 +77,7 @@ LNetEQAlloc(unsigned int count, lnet_eq_handler_t callback,
 
         /* We need count to be a power of 2 so that when eq_{enq,deq}_seq
          * overflow, they don't skip entries, so the queue has the same
 
         /* We need count to be a power of 2 so that when eq_{enq,deq}_seq
          * overflow, they don't skip entries, so the queue has the same

-         * apparant capacity at all times */
+         * apparent capacity at all times */

 
         if (count != LOWEST_BIT_SET(count)) {   /* not a power of 2 already */
                 do {                    /* knock off all but the top bit... */
 
         if (count != LOWEST_BIT_SET(count)) {   /* not a power of 2 already */
                 do {                    /* knock off all but the top bit... */


@@ -99,6 +124,16 @@ LNetEQAlloc(unsigned int count, lnet_eq_handler_t callback,
         return (0);
 }
 
         return (0);
 }
 

+/**
+ * Release the resources associated with an event queue if it's idle;
+ * otherwise do nothing and it's up to the user to try again.
+ *
+ * \param eqh A handle for the event queue to be released.
+ *
+ * \retval 0 If the EQ is not in use and freed.
+ * \retval -ENOENT If \a eqh does not point to a valid EQ.
+ * \retval -EBUSY  If the EQ is still in use by some MDs.
+ */

 int
 LNetEQFree(lnet_handle_eq_t eqh)
 {
 int
 LNetEQFree(lnet_handle_eq_t eqh)
 {


@@ -172,7 +207,22 @@ lib_get_event (lnet_eq_t *eq, lnet_event_t *ev)
         RETURN(rc);
 }
 
         RETURN(rc);
 }
 

-
+/**
+ * A nonblocking function that can be used to get the next event in an EQ.
+ * If an event handler is associated with the EQ, the handler will run before
+ * this function returns successfully. The event is removed from the queue.
+ *
+ * \param eventq A handle for the event queue.
+ * \param event On successful return (1 or -EOVERFLOW), this location will
+ * hold the next event in the EQ.
+ *
+ * \retval 0          No pending event in the EQ.
+ * \retval 1          Indicates success.
+ * \retval -ENOENT    If \a eventq does not point to a valid EQ.
+ * \retval -EOVERFLOW Indicates success (i.e., an event is returned) and that
+ * at least one event between this event and the last event obtained from the
+ * EQ has been dropped due to limited space in the EQ.
+ */

 int
 LNetEQGet (lnet_handle_eq_t eventq, lnet_event_t *event)
 {
 int
 LNetEQGet (lnet_handle_eq_t eventq, lnet_event_t *event)
 {


@@ -182,6 +232,22 @@ LNetEQGet (lnet_handle_eq_t eventq, lnet_event_t *event)
                          event, &which);
 }
 
                          event, &which);
 }
 

+/**
+ * Block the calling process until there is an event in the EQ.
+ * If an event handler is associated with the EQ, the handler will run before
+ * this function returns successfully. This function returns the next event
+ * in the EQ and removes it from the EQ.
+ *
+ * \param eventq A handle for the event queue.
+ * \param event On successful return (1 or -EOVERFLOW), this location will
+ * hold the next event in the EQ.
+ *
+ * \retval 1          Indicates success.
+ * \retval -ENOENT    If \a eventq does not point to a valid EQ.
+ * \retval -EOVERFLOW Indicates success (i.e., an event is returned) and that
+ * at least one event between this event and the last event obtained from the
+ * EQ has been dropped due to limited space in the EQ.
+ */

 int
 LNetEQWait (lnet_handle_eq_t eventq, lnet_event_t *event)
 {
 int
 LNetEQWait (lnet_handle_eq_t eventq, lnet_event_t *event)
 {


@@ -191,6 +257,32 @@ LNetEQWait (lnet_handle_eq_t eventq, lnet_event_t *event)
                          event, &which);
 }
 
                          event, &which);
 }
 

+/**
+ * Block the calling process until there's an event from a set of EQs or
+ * timeout happens.
+ *
+ * If an event handler is associated with the EQ, the handler will run before
+ * this function returns successfully, in which case the corresponding event
+ * is consumed.
+ *
+ * LNetEQPoll() provides a timeout to allow applications to poll, block for a
+ * fixed period, or block indefinitely.
+ *
+ * \param eventqs,neq An array of EQ handles, and size of the array.
+ * \param timeout_ms Time in milliseconds to wait for an event to occur on
+ * one of the EQs. The constant LNET_TIME_FOREVER can be used to indicate an
+ * infinite timeout.
+ * \param event,which On successful return (1 or -EOVERFLOW), \a event will
+ * hold the next event in the EQs, and \a which will contain the index of the
+ * EQ from which the event was taken.
+ *
+ * \retval 0          No pending event in the EQs after timeout.
+ * \retval 1          Indicates success.
+ * \retval -EOVERFLOW Indicates success (i.e., an event is returned) and that
+ * at least one event between this event and the last event obtained from the
+ * EQ indicated by \a which has been dropped due to limited space in the EQ.
+ * \retval -ENOENT    If there's an invalid handle in \a eventqs.
+ */

 int
 LNetEQPoll (lnet_handle_eq_t *eventqs, int neq, int timeout_ms,
             lnet_event_t *event, int *which)
 int
 LNetEQPoll (lnet_handle_eq_t *eventqs, int neq, int timeout_ms,
             lnet_event_t *event, int *which)

diff --git a/lnet/lnet/lib-md.c b/lnet/lnet/lib-md.c
index 7cfce40..c01f2b0 100644 (file)
--- a/lnet/lnet/lib-md.c
+++ b/lnet/lnet/lib-md.c
@@ -230,6 +230,31 @@ lnet_md_validate(lnet_md_t *umd)
         return 0;
 }
 
         return 0;
 }
 

+/**
+ * Create a memory descriptor and attach it to a ME
+ *
+ * \param meh A handle for a ME to associate the new MD with.
+ * \param umd Provides initial values for the user-visible parts of a MD.
+ * Other than its use for initialization, there is no linkage between this
+ * structure and the MD maintained by the LNet.
+ * \param unlink A flag to indicate whether the MD is automatically unlinked
+ * when it becomes inactive, either because the operation threshold drops to
+ * zero or because the available memory becomes less than \a umd.max_size.
+ * (Note that the check for unlinking a MD only occurs after the completion
+ * of a successful operation on the MD.) The value LNET_UNLINK enables auto
+ * unlinking; the value LNET_RETAIN disables it.
+ * \param handle On successful returns, a handle to the newly created MD is
+ * saved here. This handle can be used later in LNetMDUnlink().
+ *
+ * \retval 0       On success.
+ * \retval -EINVAL If \a umd is not valid.
+ * \retval -ENOMEM If new MD cannot be allocated.
+ * \retval -ENOENT Either \a meh or \a umd.eq_handle does not point to a
+ * valid object. Note that it's OK to supply a NULL \a umd.eq_handle by
+ * calling LNetInvalidateHandle() on it.
+ * \retval -EBUSY  If the ME pointed to by \a meh is already associated with
+ * a MD.
+ */

 int
 LNetMDAttach(lnet_handle_me_t meh, lnet_md_t umd,
              lnet_unlink_t unlink, lnet_handle_md_t *handle)
 int
 LNetMDAttach(lnet_handle_me_t meh, lnet_md_t umd,
              lnet_unlink_t unlink, lnet_handle_md_t *handle)


@@ -284,6 +309,22 @@ LNetMDAttach(lnet_handle_me_t meh, lnet_md_t umd,
         return (rc);
 }
 
         return (rc);
 }
 

+/**
+ * Create a "free floating" memory descriptor - a MD that is not associated
+ * with a ME. Such MDs are usually used in LNetPut() and LNetGet() operations.
+ *
+ * \param umd,unlink See the discussion for LNetMDAttach().
+ * \param handle On successful returns, a handle to the newly created MD is
+ * saved here. This handle can be used later in LNetMDUnlink(), LNetPut(),
+ * and LNetGet() operations.
+ *
+ * \retval 0       On success.
+ * \retval -EINVAL If \a umd is not valid.
+ * \retval -ENOMEM If new MD cannot be allocated.
+ * \retval -ENOENT \a umd.eq_handle does not point to a valid EQ. Note that
+ * it's OK to supply a NULL \a umd.eq_handle by calling
+ * LNetInvalidateHandle() on it.
+ */

 int
 LNetMDBind(lnet_md_t umd, lnet_unlink_t unlink, lnet_handle_md_t *handle)
 {
 int
 LNetMDBind(lnet_md_t umd, lnet_unlink_t unlink, lnet_handle_md_t *handle)
 {


@@ -322,6 +363,35 @@ LNetMDBind(lnet_md_t umd, lnet_unlink_t unlink, lnet_handle_md_t *handle)
         return (rc);
 }
 
         return (rc);
 }
 

+/**
+ * Unlink the memory descriptor from any ME it may be linked to and release
+ * the internal resources associated with it.
+ *
+ * This function does not free the memory region associated with the MD;
+ * i.e., the memory the user allocated for this MD. If the ME associated with
+ * this MD is not NULL and was created with auto unlink enabled, the ME is
+ * unlinked as well (see LNetMEAttach()).
+ *
+ * Explicitly unlinking a MD via this function call has the same behavior as
+ * a MD that has been automatically unlinked, except that no LNET_EVENT_UNLINK
+ * is generated in the latter case.
+ *
+ * An unlinked event can be reported in two ways:
+ * - If there's no pending operations on the MD, it's unlinked immediately
+ *   and an LNET_EVENT_UNLINK event is logged before this function returns.
+ * - Otherwise, the MD is only marked for deletion when this function
+ *   returns, and the unlinked event will be piggybacked on the event of
+ *   the completion of the last operation by setting the unlinked field of
+ *   the event. No dedicated LNET_EVENT_UNLINK event is generated.
+ *
+ * Note that in both cases the unlinked field of the event is always set; no
+ * more event will happen on the MD after such an event is logged.
+ *
+ * \param mdh A handle for the MD to be unlinked.
+ *
+ * \retval 0       On success.
+ * \retval -ENOENT If \a mdh does not point to a valid MD object.
+ */

 int
 LNetMDUnlink (lnet_handle_md_t mdh)
 {
 int
 LNetMDUnlink (lnet_handle_md_t mdh)
 {

diff --git a/lnet/lnet/lib-me.c b/lnet/lnet/lib-me.c
index 381e512..d62c970 100644 (file)
--- a/lnet/lnet/lib-me.c
+++ b/lnet/lnet/lib-me.c
@@ -96,6 +96,36 @@ lnet_me_match_portal(lnet_portal_t *ptl, lnet_process_id_t id,
         return 0;
 }
 
         return 0;
 }
 

+/**
+ * Create and attach a match entry to the match list of \a portal. The new
+ * ME is empty, i.e. not associated with a memory descriptor. LNetMDAttach()
+ * can be used to attach a MD to an empty ME.
+ *
+ * \param portal The portal table index where the ME should be attached.
+ * \param match_id Specifies the match criteria for the process ID of
+ * the requester. The constants LNET_PID_ANY and LNET_NID_ANY can be
+ * used to wildcard either of the identifiers in the lnet_process_id_t
+ * structure.
+ * \param match_bits,ignore_bits Specify the match criteria to apply
+ * to the match bits in the incoming request. The ignore bits are used
+ * to mask out insignificant bits in the incoming match bits. The resulting
+ * bits are then compared to the ME's match bits to determine if the
+ * incoming request meets the match criteria.
+ * \param unlink Indicates whether the ME should be unlinked when the memory
+ * descriptor associated with it is unlinked (Note that the check for
+ * unlinking a ME only occurs when the memory descriptor is unlinked.).
+ * Valid values are LNET_RETAIN and LNET_UNLINK.
+ * \param pos Indicates whether the new ME should be prepended or
+ * appended to the match list. Allowed constants: LNET_INS_BEFORE,
+ * LNET_INS_AFTER.
+ * \param handle On successful returns, a handle to the newly created ME
+ * object is saved here. This handle can be used later in LNetMEInsert(),
+ * LNetMEUnlink(), or LNetMDAttach() functions.
+ *
+ * \retval 0       On success.
+ * \retval -EINVAL If \a portal is invalid.
+ * \retval -ENOMEM If new ME object cannot be allocated.
+ */

 int
 LNetMEAttach(unsigned int portal,
              lnet_process_id_t match_id,
 int
 LNetMEAttach(unsigned int portal,
              lnet_process_id_t match_id,


@@ -148,6 +178,23 @@ LNetMEAttach(unsigned int portal,
         return 0;
 }
 
         return 0;
 }
 

+/**
+ * Create and a match entry and insert it before or after the ME pointed to by
+ * \a current_meh. The new ME is empty, i.e. not associated with a memory
+ * descriptor. LNetMDAttach() can be used to attach a MD to an empty ME.
+ *
+ * This function is identical to LNetMEAttach() except for the position
+ * where the new ME is inserted.
+ *
+ * \param current_meh A handle for a ME. The new ME will be inserted
+ * immediately before or immediately after this ME.
+ * \param match_id,match_bits,ignore_bits,unlink,pos,handle See the discussion
+ * for LNetMEAttach().
+ *
+ * \retval 0       On success.
+ * \retval -ENOMEM If new ME object cannot be allocated.
+ * \retval -ENOENT If \a current_meh does not point to a valid match entry.
+ */

 int
 LNetMEInsert(lnet_handle_me_t current_meh,
              lnet_process_id_t match_id,
 int
 LNetMEInsert(lnet_handle_me_t current_meh,
              lnet_process_id_t match_id,


@@ -207,6 +254,20 @@ LNetMEInsert(lnet_handle_me_t current_meh,
         return 0;
 }
 
         return 0;
 }
 

+/**
+ * Unlink a match entry from its match list.
+ *
+ * This operation also releases any resources associated with the ME. If a
+ * memory descriptor is attached to the ME, then it will be unlinked as well
+ * and an unlink event will be generated. It is an error to use the ME handle
+ * after calling LNetMEUnlink().
+ *
+ * \param meh A handle for the ME to be unlinked.
+ *
+ * \retval 0       On success.
+ * \retval -ENOENT If \a meh does not point to a valid ME.
+ * \see LNetMDUnlink() for the discussion on delivering unlink event.
+ */

 int
 LNetMEUnlink(lnet_handle_me_t meh)
 {
 int
 LNetMEUnlink(lnet_handle_me_t meh)
 {

diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c
index 957f958..3b87dc3 100644 (file)
--- a/lnet/lnet/lib-move.c
+++ b/lnet/lnet/lib-move.c
@@ -1567,6 +1567,34 @@ lnet_drop_delayed_put(lnet_msg_t *msg, char *reason)
         LNET_UNLOCK();
 }
 
         LNET_UNLOCK();
 }
 

+/**
+ * Turn on the lazy portal attribute. Use with caution!
+ *
+ * This portal attribute only affects incoming PUT requests to the portal,
+ * and is off by default. By default, if there's no matching MD for an
+ * incoming PUT request, it is simply dropped. With the lazy attribute on,
+ * such requests are queued indefinitely until either a matching MD is
+ * posted to the portal or the lazy attribute is turned off.
+ *
+ * It would prevent dropped requests, however it should be regarded as the
+ * last line of defense - i.e. users must keep a close watch on active
+ * buffers on a lazy portal and once it becomes too low post more buffers as
+ * soon as possible. This is because delayed requests usually have detrimental
+ * effects on underlying network connections. A few delayed requests often
+ * suffice to bring an underlying connection to a complete halt, due to flow
+ * control mechanisms.
+ *
+ * There's also a DOS attack risk. If users don't post match-all MDs on a
+ * lazy portal, a malicious peer can easily stop a service by sending some
+ * PUT requests with match bits that won't match any MD. A routed server is
+ * especially vulnerable since the connections to its neighbor routers are
+ * shared among all clients.
+ *
+ * \param portal Index of the portal to enable the lazy attribute on.
+ *
+ * \retval 0       On success.
+ * \retval -EINVAL If \a portal is not a valid index.
+ */

 int
 LNetSetLazyPortal(int portal)
 {
 int
 LNetSetLazyPortal(int portal)
 {


@@ -1584,6 +1612,15 @@ LNetSetLazyPortal(int portal)
         return 0;
 }
 
         return 0;
 }
 

+/**
+ * Turn off the lazy portal attribute. Delayed requests on the portal,
+ * if any, will be all dropped when this function returns.
+ *
+ * \param portal Index of the portal to disable the lazy attribute on.
+ *
+ * \retval 0       On success.
+ * \retval -EINVAL If \a portal is not a valid index.
+ */

 int
 LNetClearLazyPortal(int portal)
 {
 int
 LNetClearLazyPortal(int portal)
 {


@@ -2361,6 +2398,50 @@ lnet_parse(lnet_ni_t *ni, lnet_hdr_t *hdr, lnet_nid_t from_nid,
         return 0;
 }
 
         return 0;
 }
 

+/**
+ * Initiate an asynchronous PUT operation.
+ *
+ * There are several events associated with a PUT: completion of the send on
+ * the initiator node (LNET_EVENT_SEND), and when the send completes
+ * successfully, the receipt of an acknowledgment (LNET_EVENT_ACK) indicating
+ * that the operation was accepted by the target. The event LNET_EVENT_PUT is
+ * used at the target node to indicate the completion of incoming data
+ * delivery.
+ *
+ * The local events will be logged in the EQ associated with the MD pointed to
+ * by \a mdh handle. Using a MD without an associated EQ results in these
+ * events being discarded. In this case, the caller must have another
+ * mechanism (e.g., a higher level protocol) for determining when it is safe
+ * to modify the memory region associated with the MD.
+ *
+ * Note that LNet does not guarantee the order of LNET_EVENT_SEND and
+ * LNET_EVENT_ACK, though intuitively ACK should happen after SEND.
+ *
+ * \param self Indicates the NID of a local interface through which to send
+ * the PUT request. Use LNET_NID_ANY to let LNet choose one by itself.
+ * \param mdh A handle for the MD that describes the memory to be sent. The MD
+ * must be "free floating" (See LNetMDBind()).
+ * \param ack Controls whether an acknowledgment is requested.
+ * Acknowledgments are only sent when they are requested by the initiating
+ * process and the target MD enables them.
+ * \param target A process identifier for the target process.
+ * \param portal The index in the \a target's portal table.
+ * \param match_bits The match bits to use for MD selection at the target
+ * process.
+ * \param offset The offset into the target MD (only used when the target
+ * MD has the LNET_MD_MANAGE_REMOTE option set).
+ * \param hdr_data 64 bits of user data that can be included in the message
+ * header. This data is written to an event queue entry at the target if an
+ * EQ is present on the matching MD.
+ *
+ * \retval  0      Success, and only in this case events will be generated
+ * and logged to EQ (if it exists).
+ * \retval -EIO    Simulated failure.
+ * \retval -ENOMEM Memory allocation failure.
+ * \retval -ENOENT Invalid MD object.
+ *
+ * \see lnet_event_t::hdr_data and lnet_event_kind_t.
+ */

 int
 LNetPut(lnet_nid_t self, lnet_handle_md_t mdh, lnet_ack_req_t ack,
         lnet_process_id_t target, unsigned int portal,
 int
 LNetPut(lnet_nid_t self, lnet_handle_md_t mdh, lnet_ack_req_t ack,
         lnet_process_id_t target, unsigned int portal,


@@ -2548,6 +2629,26 @@ lnet_set_reply_msg_len(lnet_ni_t *ni, lnet_msg_t *reply, unsigned int len)
         reply->msg_ev.mlength = len;
 }
 
         reply->msg_ev.mlength = len;
 }
 

+/**
+ * Initiate an asynchronous GET operation.
+ *
+ * On the initiator node, an LNET_EVENT_SEND is logged when the GET request
+ * is sent, and an LNET_EVENT_REPLY is logged when the data returned from
+ * the target node in the REPLY has been written to local MD.
+ *
+ * On the target node, an LNET_EVENT_GET is logged when the GET request
+ * arrives and is accepted into a MD.
+ *
+ * \param self,target,portal,match_bits,offset See the discussion in LNetPut().
+ * \param mdh A handle for the MD that describes the memory into which the
+ * requested data will be received. The MD must be "free floating" (See LNetMDBind()).
+ *
+ * \retval  0      Success, and only in this case events will be generated
+ * and logged to EQ (if it exists) of the MD.
+ * \retval -EIO    Simulated failure.
+ * \retval -ENOMEM Memory allocation failure.
+ * \retval -ENOENT Invalid MD object.
+ */

 int
 LNetGet(lnet_nid_t self, lnet_handle_md_t mdh, 
         lnet_process_id_t target, unsigned int portal, 
 int
 LNetGet(lnet_nid_t self, lnet_handle_md_t mdh, 
         lnet_process_id_t target, unsigned int portal, 


@@ -2639,6 +2740,20 @@ LNetGet(lnet_nid_t self, lnet_handle_md_t mdh,
         return 0;
 }
 
         return 0;
 }
 

+/**
+ * Calculate distance to node at \a dstnid.
+ *
+ * \param dstnid Target NID.
+ * \param srcnidp If not NULL, NID of the local interface to reach \a dstnid
+ * is saved here.
+ * \param orderp If not NULL, order of the route to reach \a dstnid is saved
+ * here.
+ *
+ * \retval 0 If \a dstnid belongs to a local interface, and reserved option
+ * local_nid_dist_zero is set, which is the default.
+ * \retval positives Distance to target NID, i.e. number of hops plus one.
+ * \retval -EHOSTUNREACH If \a dstnid is not reachable.
+ */

 int
 LNetDist (lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp)
 {
 int
 LNetDist (lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp)
 {


@@ -2720,6 +2835,23 @@ LNetDist (lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp)
         return -EHOSTUNREACH;
 }
 
         return -EHOSTUNREACH;
 }
 

+/**
+ * Set the number of asynchronous messages expected from a target process.
+ *
+ * This function is only meaningful for userspace callers. It's a no-op when
+ * called from kernel.
+ *
+ * Asynchronous messages are those that can come from a target when the
+ * userspace process is not waiting for IO to complete; e.g., AST callbacks
+ * from Lustre servers. Specifying the expected number of such messages
+ * allows them to be eagerly received when user process is not running in
+ * LNet; otherwise network errors may occur.
+ *
+ * \param id Process ID of the target process.
+ * \param nasync Number of asynchronous messages expected from the target.
+ *
+ * \return 0 on success, and an error code otherwise.
+ */

 int
 LNetSetAsync(lnet_process_id_t id, int nasync)
 {
 int
 LNetSetAsync(lnet_process_id_t id, int nasync)
 {


@@ -2737,7 +2869,6 @@ LNetSetAsync(lnet_process_id_t id, int nasync)
         int               rc2;
 
         /* Target on a local network? */
         int               rc2;
 
         /* Target on a local network? */

-

         ni = lnet_net2ni(LNET_NIDNET(id.nid));
         if (ni != NULL) {
                 if (ni->ni_lnd->lnd_setasync != NULL) 
         ni = lnet_net2ni(LNET_NIDNET(id.nid));
         if (ni != NULL) {
                 if (ni->ni_lnd->lnd_setasync != NULL) 

diff --git a/lnet/lnet/lib-msg.c b/lnet/lnet/lib-msg.c
index 3383d91..4ab5fa4 100644 (file)
--- a/lnet/lnet/lib-msg.c
+++ b/lnet/lnet/lib-msg.c
@@ -72,7 +72,7 @@ lnet_enq_event_locked (lnet_eq_t *eq, lnet_event_t *ev)
 
         /* There is no race since both event consumers and event producers
          * take the LNET_LOCK, so we don't screw around with memory
 
         /* There is no race since both event consumers and event producers
          * take the LNET_LOCK, so we don't screw around with memory

-         * barriers, setting the sequence number last or wierd structure
+         * barriers, setting the sequence number last or weird structure

          * layout assertions. */
         *eq_slot = *ev;
 
          * layout assertions. */
         *eq_slot = *ev;