Whamcloud - gitweb
LU-5620 ptlrpc: Add QoS for opcode in NRS-TBF
[fs/lustre-release.git] / lustre / include / lustre_net.h
index 9a0b67e..3c1f595 100644 (file)
  *
  * You should have received a copy of the GNU General Public License
  * version 2 along with this program; If not, see
- * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
- *
- * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
- * CA 95054 USA or visit www.sun.com if you need additional information or
- * have any questions.
+ * http://www.gnu.org/licenses/gpl-2.0.html
  *
  * GPL HEADER END
  */
@@ -27,7 +23,7 @@
  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  * Use is subject to license terms.
  *
- * Copyright (c) 2010, 2014, Intel Corporation.
+ * Copyright (c) 2010, 2016, Intel Corporation.
  */
 /*
  * This file is part of Lustre, http://www.lustre.org/
 #define PTLRPC_MD_OPTIONS  0
 
 /**
- * Max # of bulk operations in one request.
+ * log2 max # of bulk operations in one request: 2=4MB/RPC, 5=32MB/RPC, ...
  * In order for the client and server to properly negotiate the maximum
  * possible transfer size, PTLRPC_BULK_OPS_COUNT must be a power-of-two
  * value.  The client is free to limit the actual RPC size for any bulk
- * transfer via cl_max_pages_per_rpc to some non-power-of-two value. */
-#define PTLRPC_BULK_OPS_BITS   2
+ * transfer via cl_max_pages_per_rpc to some non-power-of-two value.
+ * NOTE: This is limited to 16 (=64GB RPCs) by IOOBJ_MAX_BRW_BITS. */
+#define PTLRPC_BULK_OPS_BITS   4
+#if PTLRPC_BULK_OPS_BITS > 16
+#error "More than 65536 BRW RPCs not allowed by IOOBJ_MAX_BRW_BITS."
+#endif
 #define PTLRPC_BULK_OPS_COUNT  (1U << PTLRPC_BULK_OPS_BITS)
 /**
  * PTLRPC_BULK_OPS_MASK is for the convenience of the client only, and
  */
 #define PTLRPC_MAX_BRW_BITS    (LNET_MTU_BITS + PTLRPC_BULK_OPS_BITS)
 #define PTLRPC_MAX_BRW_SIZE    (1 << PTLRPC_MAX_BRW_BITS)
-#define PTLRPC_MAX_BRW_PAGES   (PTLRPC_MAX_BRW_SIZE >> PAGE_CACHE_SHIFT)
+#define PTLRPC_MAX_BRW_PAGES   (PTLRPC_MAX_BRW_SIZE >> PAGE_SHIFT)
 
 #define ONE_MB_BRW_SIZE                (1 << LNET_MTU_BITS)
 #define MD_MAX_BRW_SIZE                (1 << LNET_MTU_BITS)
-#define MD_MAX_BRW_PAGES       (MD_MAX_BRW_SIZE >> PAGE_CACHE_SHIFT)
+#define MD_MAX_BRW_PAGES       (MD_MAX_BRW_SIZE >> PAGE_SHIFT)
 #define DT_MAX_BRW_SIZE                PTLRPC_MAX_BRW_SIZE
-#define DT_MAX_BRW_PAGES       (DT_MAX_BRW_SIZE >> PAGE_CACHE_SHIFT)
+#define DT_MAX_BRW_PAGES       (DT_MAX_BRW_SIZE >> PAGE_SHIFT)
 #define OFD_MAX_BRW_SIZE       (1 << LNET_MTU_BITS)
 
 /* When PAGE_SIZE is a constant, we can check our arithmetic here with cpp! */
 #if ((PTLRPC_MAX_BRW_PAGES & (PTLRPC_MAX_BRW_PAGES - 1)) != 0)
 # error "PTLRPC_MAX_BRW_PAGES isn't a power of two"
 #endif
-#if (PTLRPC_MAX_BRW_SIZE != (PTLRPC_MAX_BRW_PAGES * PAGE_CACHE_SIZE))
-# error "PTLRPC_MAX_BRW_SIZE isn't PTLRPC_MAX_BRW_PAGES * PAGE_CACHE_SIZE"
+#if (PTLRPC_MAX_BRW_SIZE != (PTLRPC_MAX_BRW_PAGES * PAGE_SIZE))
+# error "PTLRPC_MAX_BRW_SIZE isn't PTLRPC_MAX_BRW_PAGES * PAGE_SIZE"
 #endif
 #if (PTLRPC_MAX_BRW_SIZE > LNET_MTU * PTLRPC_BULK_OPS_COUNT)
 # error "PTLRPC_MAX_BRW_SIZE too big"
   */
  /* depress threads factor for VM with small memory size */
 #define OSS_THR_FACTOR         min_t(int, 8, \
-                               NUM_CACHEPAGES >> (28 - PAGE_CACHE_SHIFT))
+                               NUM_CACHEPAGES >> (28 - PAGE_SHIFT))
 #define OSS_NTHRS_INIT         (PTLRPC_NTHRS_INIT + 1)
 #define OSS_NTHRS_BASE         64
-#define OSS_NTHRS_MAX          512
 
 /* threads for handling "create" request */
 #define OSS_CR_THR_FACTOR      1
@@ -605,6 +604,7 @@ struct ptlrpc_request_set {
        set_producer_func       set_producer;
        /** opaq argument passed to the producer callback */
        void                    *set_producer_arg;
+       unsigned int             set_allow_intr:1;
 };
 
 /**
@@ -668,6 +668,10 @@ struct ptlrpc_reply_state {
         unsigned long          rs_committed:1;/* the transaction was committed
                                                  and the rs was dispatched
                                                  by ptlrpc_commit_replies */
+       atomic_t                rs_refcount;    /* number of users */
+       /** Number of locks awaiting client ACK */
+       int                     rs_nlocks;
+
         /** Size of the state */
         int                    rs_size;
         /** opcode */
@@ -679,43 +683,41 @@ struct ptlrpc_reply_state {
        struct obd_export     *rs_export;
        struct ptlrpc_service_part *rs_svcpt;
        /** Lnet metadata handle for the reply */
-       lnet_handle_md_t       rs_md_h;
-       atomic_t               rs_refcount;
+       lnet_handle_md_t        rs_md_h;
 
        /** Context for the sevice thread */
-       struct ptlrpc_svc_ctx *rs_svc_ctx;
+       struct ptlrpc_svc_ctx   *rs_svc_ctx;
        /** Reply buffer (actually sent to the client), encoded if needed */
-       struct lustre_msg     *rs_repbuf;       /* wrapper */
-        /** Size of the reply buffer */
-        int                    rs_repbuf_len;   /* wrapper buf length */
-        /** Size of the reply message */
-        int                    rs_repdata_len;  /* wrapper msg length */
-        /**
-         * Actual reply message. Its content is encrupted (if needed) to
-         * produce reply buffer for actual sending. In simple case
-         * of no network encryption we jus set \a rs_repbuf to \a rs_msg
-         */
-        struct lustre_msg     *rs_msg;          /* reply message */
-
-        /** Number of locks awaiting client ACK */
-        int                    rs_nlocks;
-        /** Handles of locks awaiting client reply ACK */
-        struct lustre_handle   rs_locks[RS_MAX_LOCKS];
-        /** Lock modes of locks in \a rs_locks */
-        ldlm_mode_t            rs_modes[RS_MAX_LOCKS];
+       struct lustre_msg       *rs_repbuf;     /* wrapper */
+       /** Size of the reply buffer */
+       int                     rs_repbuf_len;  /* wrapper buf length */
+       /** Size of the reply message */
+       int                     rs_repdata_len; /* wrapper msg length */
+       /**
+        * Actual reply message. Its content is encrupted (if needed) to
+        * produce reply buffer for actual sending. In simple case
+        * of no network encryption we jus set \a rs_repbuf to \a rs_msg
+        */
+       struct lustre_msg       *rs_msg;        /* reply message */
+
+       /** Handles of locks awaiting client reply ACK */
+       struct lustre_handle    rs_locks[RS_MAX_LOCKS];
+       /** Lock modes of locks in \a rs_locks */
+       enum ldlm_mode          rs_modes[RS_MAX_LOCKS];
 };
 
 struct ptlrpc_thread;
 
 /** RPC stages */
 enum rq_phase {
-        RQ_PHASE_NEW            = 0xebc0de00,
-        RQ_PHASE_RPC            = 0xebc0de01,
-        RQ_PHASE_BULK           = 0xebc0de02,
-        RQ_PHASE_INTERPRET      = 0xebc0de03,
-        RQ_PHASE_COMPLETE       = 0xebc0de04,
-        RQ_PHASE_UNREGISTERING  = 0xebc0de05,
-        RQ_PHASE_UNDEFINED      = 0xebc0de06
+       RQ_PHASE_NEW            = 0xebc0de00,
+       RQ_PHASE_RPC            = 0xebc0de01,
+       RQ_PHASE_BULK           = 0xebc0de02,
+       RQ_PHASE_INTERPRET      = 0xebc0de03,
+       RQ_PHASE_COMPLETE       = 0xebc0de04,
+       RQ_PHASE_UNREG_RPC      = 0xebc0de05,
+       RQ_PHASE_UNREG_BULK     = 0xebc0de06,
+       RQ_PHASE_UNDEFINED      = 0xebc0de07
 };
 
 /** Type of request interpreter call-back */
@@ -740,7 +742,7 @@ struct ptlrpc_request_pool {
        /** Maximum message size that would fit into a rquest from this pool */
        int                     prp_rq_size;
        /** Function to allocate more requests for this pool */
-       void (*prp_populate)(struct ptlrpc_request_pool *, int);
+       int (*prp_populate)(struct ptlrpc_request_pool *, int);
 };
 
 struct lu_context;
@@ -790,6 +792,8 @@ struct ptlrpc_cli_req {
        time_t                           cr_reply_deadline;
        /** when req bulk unlink must finish. */
        time_t                           cr_bulk_deadline;
+       /** when req unlink must finish. */
+       time_t                           cr_req_deadline;
        /** Portal to which this request would be sent */
        short                            cr_req_ptl;
        /** Portal where to wait for reply and where reply would be sent */
@@ -827,6 +831,8 @@ struct ptlrpc_cli_req {
        union ptlrpc_async_args          cr_async_args;
        /** Opaq data for replay and commit callbacks. */
        void                            *cr_cb_data;
+       /** Link to the imp->imp_unreplied_list */
+       struct list_head                 cr_unreplied_list;
        /**
         * Commit callback, called when request is committed and about to be
         * freed.
@@ -846,6 +852,7 @@ struct ptlrpc_cli_req {
 #define rq_real_sent           rq_cli.cr_sent_out
 #define rq_reply_deadline      rq_cli.cr_reply_deadline
 #define rq_bulk_deadline       rq_cli.cr_bulk_deadline
+#define rq_req_deadline                rq_cli.cr_req_deadline
 #define rq_nr_resend           rq_cli.cr_resend_nr
 #define rq_request_portal      rq_cli.cr_req_ptl
 #define rq_reply_portal                rq_cli.cr_rep_ptl
@@ -865,6 +872,7 @@ struct ptlrpc_cli_req {
 #define rq_resend_cb           rq_cli.cr_resend_cb
 #define rq_async_args          rq_cli.cr_async_args
 #define rq_cb_data             rq_cli.cr_cb_data
+#define rq_unreplied_list      rq_cli.cr_unreplied_list
 #define rq_commit_cb           rq_cli.cr_commit_cb
 #define rq_replay_cb           rq_cli.cr_replay_cb
 
@@ -886,7 +894,7 @@ struct ptlrpc_srv_req {
        /** history sequence # */
        __u64                            sr_hist_seq;
        /** the index of service's srv_at_array into which request is linked */
-       time_t                           sr_at_index;
+       __u32                            sr_at_index;
        /** authed uid */
        uid_t                            sr_auth_uid;
        /** authed uid mapped to */
@@ -907,7 +915,7 @@ struct ptlrpc_srv_req {
        struct ptlrpc_svc_ctx           *sr_svc_ctx;
        /** (server side), pointed directly into req buffer */
        struct ptlrpc_user_desc         *sr_user_desc;
-       /** separated reply state */
+       /** separated reply state, may be vmalloc'd */
        struct ptlrpc_reply_state       *sr_reply_state;
        /** server-side hp handlers */
        struct ptlrpc_hpreq_ops         *sr_ops;
@@ -986,7 +994,8 @@ struct ptlrpc_request {
                 * status */
                rq_allow_replay:1,
                /* bulk request, sent to server, but uncommitted */
-               rq_unstable:1;
+               rq_unstable:1,
+               rq_allow_intr:1;
        /** @} */
 
        /** server-side flags @{ */
@@ -1028,7 +1037,7 @@ struct ptlrpc_request {
        /** bulk match bits */
        __u64                            rq_mbits;
        /**
-        * List item to for replay list. Not yet commited requests get linked
+        * List item to for replay list. Not yet committed requests get linked
         * there.
         * Also see \a rq_replay comment above.
         * It's also link chain on obd_export::exp_req_replay_queue
@@ -1053,7 +1062,6 @@ struct ptlrpc_request {
                                  rq_bulk_write:1,    /* request bulk write */
                                  /* server authentication flags */
                                  rq_auth_gss:1,      /* authenticated by gss */
-                                 rq_auth_remote:1,   /* authed as remote user */
                                  rq_auth_usr_root:1, /* authed as root */
                                  rq_auth_usr_mdt:1,  /* authed as mdt */
                                  rq_auth_usr_ost:1,  /* authed as ost */
@@ -1067,9 +1075,9 @@ struct ptlrpc_request {
 
 
        /** various buffer pointers */
-       struct lustre_msg               *rq_reqbuf;      /**< req wrapper */
-       char                            *rq_repbuf;      /**< rep buffer */
-       struct lustre_msg               *rq_repdata;     /**< rep wrapper msg */
+       struct lustre_msg               *rq_reqbuf;  /**< req wrapper, vmalloc*/
+       char                            *rq_repbuf;  /**< rep buffer, vmalloc */
+       struct lustre_msg               *rq_repdata; /**< rep wrapper msg */
        /** only in priv mode */
        struct lustre_msg               *rq_clrbuf;
         int                      rq_reqbuf_len;  /* req wrapper buf len */
@@ -1081,7 +1089,6 @@ struct ptlrpc_request {
 
        /** early replies go to offset 0, regular replies go after that */
        unsigned int                     rq_reply_off;
-
        /** @} */
 
        /** Fields that help to see if request and reply were swabbed or not */
@@ -1098,6 +1105,8 @@ struct ptlrpc_request {
        lnet_nid_t                       rq_self;
        /** Peer description (the other side) */
        lnet_process_id_t                rq_peer;
+       /** Descriptor for the NID from which the peer sent the request. */
+       lnet_process_id_t                rq_source;
        /**
         * service time estimate (secs)
         * If the request is not served by this time, it is marked as timed out.
@@ -1220,22 +1229,24 @@ static inline void lustre_set_rep_swabbed(struct ptlrpc_request *req,
 static inline const char *
 ptlrpc_phase2str(enum rq_phase phase)
 {
-        switch (phase) {
-        case RQ_PHASE_NEW:
-                return "New";
-        case RQ_PHASE_RPC:
-                return "Rpc";
-        case RQ_PHASE_BULK:
-                return "Bulk";
-        case RQ_PHASE_INTERPRET:
-                return "Interpret";
-        case RQ_PHASE_COMPLETE:
-                return "Complete";
-        case RQ_PHASE_UNREGISTERING:
-                return "Unregistering";
-        default:
-                return "?Phase?";
-        }
+       switch (phase) {
+       case RQ_PHASE_NEW:
+               return "New";
+       case RQ_PHASE_RPC:
+               return "Rpc";
+       case RQ_PHASE_BULK:
+               return "Bulk";
+       case RQ_PHASE_INTERPRET:
+               return "Interpret";
+       case RQ_PHASE_COMPLETE:
+               return "Complete";
+       case RQ_PHASE_UNREG_RPC:
+               return "UnregRPC";
+       case RQ_PHASE_UNREG_BULK:
+               return "UnregBULK";
+       default:
+               return "?Phase?";
+       }
 }
 
 /**
@@ -1256,18 +1267,18 @@ ptlrpc_rqphase2str(struct ptlrpc_request *req)
 #define FLAG(field, str) (field ? str : "")
 
 /** Convert bit flags into a string */
-#define DEBUG_REQ_FLAGS(req)                                                    \
-        ptlrpc_rqphase2str(req),                                                \
-        FLAG(req->rq_intr, "I"), FLAG(req->rq_replied, "R"),                    \
-        FLAG(req->rq_err, "E"),                                                 \
-        FLAG(req->rq_timedout, "X") /* eXpired */, FLAG(req->rq_resend, "S"),   \
-        FLAG(req->rq_restart, "T"), FLAG(req->rq_replay, "P"),                  \
-        FLAG(req->rq_no_resend, "N"),                                           \
-        FLAG(req->rq_waiting, "W"),                                             \
-        FLAG(req->rq_wait_ctx, "C"), FLAG(req->rq_hp, "H"),                     \
-        FLAG(req->rq_committed, "M")
-
-#define REQ_FLAGS_FMT "%s:%s%s%s%s%s%s%s%s%s%s%s%s"
+#define DEBUG_REQ_FLAGS(req)                                                   \
+       ptlrpc_rqphase2str(req),                                               \
+       FLAG(req->rq_intr, "I"), FLAG(req->rq_replied, "R"),                   \
+       FLAG(req->rq_err, "E"), FLAG(req->rq_net_err, "e"),                    \
+       FLAG(req->rq_timedout, "X") /* eXpired */, FLAG(req->rq_resend, "S"),  \
+       FLAG(req->rq_restart, "T"), FLAG(req->rq_replay, "P"),                 \
+       FLAG(req->rq_no_resend, "N"),                                          \
+       FLAG(req->rq_waiting, "W"),                                            \
+       FLAG(req->rq_wait_ctx, "C"), FLAG(req->rq_hp, "H"),                    \
+       FLAG(req->rq_committed, "M")
+
+#define REQ_FLAGS_FMT "%s:%s%s%s%s%s%s%s%s%s%s%s%s%s"
 
 void _debug_req(struct ptlrpc_request *req,
                 struct libcfs_debug_msg_data *data, const char *fmt, ...)
@@ -1294,7 +1305,7 @@ do {                                                                          \
 #define DEBUG_REQ(level, req, fmt, args...)                                   \
 do {                                                                          \
         if ((level) & (D_ERROR | D_WARNING)) {                                \
-                static cfs_debug_limit_state_t cdls;                          \
+               static struct cfs_debug_limit_state cdls;                     \
                 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, level, &cdls);            \
                 debug_req(&msgdata, level, &cdls, req, "@@@ "fmt" ", ## args);\
         } else {                                                              \
@@ -1462,12 +1473,12 @@ struct ptlrpc_bulk_desc {
                         * encrypt iov, size is either 0 or bd_iov_count.
                         */
                        lnet_kiov_t *bd_enc_vec;
-                       lnet_kiov_t bd_vec[0];
+                       lnet_kiov_t *bd_vec;
                } bd_kiov;
 
                struct {
                        struct kvec *bd_enc_kvec;
-                       struct kvec bd_kvec[0];
+                       struct kvec *bd_kvec;
                } bd_kvec;
        } bd_u;
 
@@ -1496,30 +1507,30 @@ enum {
  * Definition of server service thread structure
  */
 struct ptlrpc_thread {
-        /**
-         * List of active threads in svc->srv_threads
-         */
+       /**
+        * List of active threads in svc->srv_threads
+        */
        struct list_head t_link;
-        /**
-         * thread-private data (preallocated memory)
-         */
-        void *t_data;
-        __u32 t_flags;
-        /**
-         * service thread index, from ptlrpc_start_threads
-         */
-        unsigned int t_id;
-        /**
-         * service thread pid
-         */
-        pid_t t_pid;
-        /**
-         * put watchdog in the structure per thread b=14840
-         */
-        struct lc_watchdog *t_watchdog;
-        /**
-         * the svc this thread belonged to b=18582
-         */
+       /**
+        * thread-private data (preallocated vmalloc'd memory)
+        */
+       void *t_data;
+       __u32 t_flags;
+       /**
+        * service thread index, from ptlrpc_start_threads
+        */
+       unsigned int t_id;
+       /**
+        * service thread pid
+        */
+       pid_t t_pid;
+       /**
+        * put watchdog in the structure per thread b=14840
+        */
+       struct lc_watchdog *t_watchdog;
+       /**
+        * the svc this thread belonged to b=18582
+        */
        struct ptlrpc_service_part      *t_svcpt;
        wait_queue_head_t               t_ctl_waitq;
        struct lu_env                   *t_env;
@@ -1872,34 +1883,38 @@ struct ptlrpcd_ctl {
         * Stop completion.
         */
        struct completion               pc_finishing;
-        /**
-         * Thread requests set.
-         */
-        struct ptlrpc_request_set  *pc_set;
-        /**
+       /**
+        * Thread requests set.
+        */
+       struct ptlrpc_request_set       *pc_set;
+       /**
         * Thread name used in kthread_run()
-         */
-        char                        pc_name[16];
-        /**
-         * Environment for request interpreters to run in.
-         */
-        struct lu_env               pc_env;
+        */
+       char                            pc_name[16];
+       /**
+        * CPT the thread is bound on.
+        */
+       int                             pc_cpt;
         /**
          * Index of ptlrpcd thread in the array.
          */
-        int                         pc_index;
-        /**
-         * Number of the ptlrpcd's partners.
-         */
-        int                         pc_npartners;
-        /**
-         * Pointer to the array of partners' ptlrpcd_ctl structure.
-         */
-        struct ptlrpcd_ctl        **pc_partners;
-        /**
-         * Record the partner index to be processed next.
-         */
-        int                         pc_cursor;
+       int                             pc_index;
+       /**
+        * Pointer to the array of partners' ptlrpcd_ctl structure.
+        */
+       struct ptlrpcd_ctl              **pc_partners;
+       /**
+        * Number of the ptlrpcd's partners.
+        */
+       int                             pc_npartners;
+       /**
+        * Record the partner index to be processed next.
+        */
+       int                             pc_cursor;
+       /**
+        * Error code if the thread failed to fully start.
+        */
+       int                             pc_error;
 };
 
 /* Bits for pc_flags */
@@ -1922,10 +1937,6 @@ enum ptlrpcd_ctl_flags {
          * This is a recovery ptlrpc thread.
          */
         LIOD_RECOVERY    = 1 << 3,
-        /**
-         * The ptlrpcd is bound to some CPU core.
-         */
-        LIOD_BIND        = 1 << 4,
 };
 
 /**
@@ -2032,17 +2043,16 @@ int ptlrpc_unregister_bulk(struct ptlrpc_request *req, int async);
 static inline int ptlrpc_client_bulk_active(struct ptlrpc_request *req)
 {
        struct ptlrpc_bulk_desc *desc;
-        int                      rc;
+       int rc;
 
-        LASSERT(req != NULL);
+       LASSERT(req != NULL);
        desc = req->rq_bulk;
 
-        if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_BULK_UNLINK) &&
-            req->rq_bulk_deadline > cfs_time_current_sec())
-                return 1;
+       if (req->rq_bulk_deadline > cfs_time_current_sec())
+               return 1;
 
-        if (!desc)
-                return 0;
+       if (!desc)
+               return 0;
 
        spin_lock(&desc->bd_lock);
        rc = desc->bd_md_count;
@@ -2072,7 +2082,8 @@ void ptlrpc_request_committed(struct ptlrpc_request *req, int force);
 void ptlrpc_init_client(int req_portal, int rep_portal, char *name,
                         struct ptlrpc_client *);
 void ptlrpc_cleanup_client(struct obd_import *imp);
-struct ptlrpc_connection *ptlrpc_uuid_to_connection(struct obd_uuid *uuid);
+struct ptlrpc_connection *ptlrpc_uuid_to_connection(struct obd_uuid *uuid,
+                                                   lnet_nid_t nid4refnet);
 
 int ptlrpc_queue_wait(struct ptlrpc_request *req);
 int ptlrpc_replay_req(struct ptlrpc_request *req);
@@ -2093,11 +2104,11 @@ void ptlrpc_set_destroy(struct ptlrpc_request_set *);
 void ptlrpc_set_add_req(struct ptlrpc_request_set *, struct ptlrpc_request *);
 
 void ptlrpc_free_rq_pool(struct ptlrpc_request_pool *pool);
-void ptlrpc_add_rqs_to_pool(struct ptlrpc_request_pool *pool, int num_rq);
+int ptlrpc_add_rqs_to_pool(struct ptlrpc_request_pool *pool, int num_rq);
 
 struct ptlrpc_request_pool *
 ptlrpc_init_rq_pool(int, int,
-                    void (*populate_pool)(struct ptlrpc_request_pool *, int));
+                   int (*populate_pool)(struct ptlrpc_request_pool *, int));
 
 void ptlrpc_at_set_req_timeout(struct ptlrpc_request *req);
 struct ptlrpc_request *ptlrpc_request_alloc(struct obd_import *imp,
@@ -2114,13 +2125,6 @@ struct ptlrpc_request *ptlrpc_request_alloc_pack(struct obd_import *imp,
 int ptlrpc_request_bufs_pack(struct ptlrpc_request *request,
                              __u32 version, int opcode, char **bufs,
                              struct ptlrpc_cli_ctx *ctx);
-struct ptlrpc_request *ptlrpc_prep_req(struct obd_import *imp, __u32 version,
-                                       int opcode, int count, __u32 *lengths,
-                                       char **bufs);
-struct ptlrpc_request *ptlrpc_prep_req_pool(struct obd_import *imp,
-                                             __u32 version, int opcode,
-                                            int count, __u32 *lengths, char **bufs,
-                                            struct ptlrpc_request_pool *pool);
 void ptlrpc_req_finished(struct ptlrpc_request *request);
 void ptlrpc_req_finished_with_imp_lock(struct ptlrpc_request *request);
 struct ptlrpc_request *ptlrpc_request_addref(struct ptlrpc_request *req);
@@ -2157,7 +2161,7 @@ static inline void ptlrpc_release_bulk_page_pin(struct ptlrpc_bulk_desc *desc)
        int i;
 
        for (i = 0; i < desc->bd_iov_count ; i++)
-               page_cache_release(BD_GET_KIOV(desc, i).kiov_page);
+               put_page(BD_GET_KIOV(desc, i).kiov_page);
 }
 
 static inline void ptlrpc_release_bulk_noop(struct ptlrpc_bulk_desc *desc)
@@ -2281,8 +2285,8 @@ int ptlrpc_init_import(struct obd_import *imp);
 int ptlrpc_disconnect_import(struct obd_import *imp, int noclose);
 int ptlrpc_import_recovery_state_machine(struct obd_import *imp);
 void deuuidify(char *uuid, const char *prefix, char **uuid_start,
-               int *uuid_len);
-
+              int *uuid_len);
+void ptlrpc_import_enter_resend(struct obd_import *imp);
 /* ptlrpc/pack_generic.c */
 int ptlrpc_reconnect_import(struct obd_import *imp);
 /** @} */
@@ -2356,11 +2360,7 @@ __u32 lustre_msg_get_service_time(struct lustre_msg *msg);
 char *lustre_msg_get_jobid(struct lustre_msg *msg);
 __u32 lustre_msg_get_cksum(struct lustre_msg *msg);
 __u64 lustre_msg_get_mbits(struct lustre_msg *msg);
-#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 7, 53, 0)
-__u32 lustre_msg_calc_cksum(struct lustre_msg *msg, int compat18);
-#else
 __u32 lustre_msg_calc_cksum(struct lustre_msg *msg);
-#endif
 void lustre_msg_set_handle(struct lustre_msg *msg,struct lustre_handle *handle);
 void lustre_msg_set_type(struct lustre_msg *msg, __u32 type);
 void lustre_msg_set_opc(struct lustre_msg *msg, __u32 opc);
@@ -2429,13 +2429,20 @@ ptlrpc_rqphase_move(struct ptlrpc_request *req, enum rq_phase new_phase)
        if (req->rq_phase == new_phase)
                return;
 
-       if (new_phase == RQ_PHASE_UNREGISTERING) {
+       if (new_phase == RQ_PHASE_UNREG_RPC ||
+           new_phase == RQ_PHASE_UNREG_BULK) {
+               /* No embedded unregistering phases */
+               if (req->rq_phase == RQ_PHASE_UNREG_RPC ||
+                   req->rq_phase == RQ_PHASE_UNREG_BULK)
+                       return;
+
                req->rq_next_phase = req->rq_phase;
                if (req->rq_import)
                        atomic_inc(&req->rq_import->imp_unregistering);
        }
 
-       if (req->rq_phase == RQ_PHASE_UNREGISTERING) {
+       if (req->rq_phase == RQ_PHASE_UNREG_RPC ||
+           req->rq_phase == RQ_PHASE_UNREG_BULK) {
                if (req->rq_import)
                        atomic_dec(&req->rq_import->imp_unregistering);
        }
@@ -2452,9 +2459,6 @@ ptlrpc_rqphase_move(struct ptlrpc_request *req, enum rq_phase new_phase)
 static inline int
 ptlrpc_client_early(struct ptlrpc_request *req)
 {
-        if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_REPL_UNLINK) &&
-            req->rq_reply_deadline > cfs_time_current_sec())
-                return 0;
         return req->rq_early;
 }
 
@@ -2464,20 +2468,18 @@ ptlrpc_client_early(struct ptlrpc_request *req)
 static inline int
 ptlrpc_client_replied(struct ptlrpc_request *req)
 {
-        if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_REPL_UNLINK) &&
-            req->rq_reply_deadline > cfs_time_current_sec())
-                return 0;
-        return req->rq_replied;
+       if (req->rq_reply_deadline > cfs_time_current_sec())
+               return 0;
+       return req->rq_replied;
 }
 
 /** Returns true if request \a req is in process of receiving server reply */
 static inline int
 ptlrpc_client_recv(struct ptlrpc_request *req)
 {
-        if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_REPL_UNLINK) &&
-            req->rq_reply_deadline > cfs_time_current_sec())
-                return 1;
-        return req->rq_receiving_reply;
+       if (req->rq_reply_deadline > cfs_time_current_sec())
+               return 1;
+       return req->rq_receiving_reply;
 }
 
 static inline int
@@ -2486,11 +2488,15 @@ ptlrpc_client_recv_or_unlink(struct ptlrpc_request *req)
        int rc;
 
        spin_lock(&req->rq_lock);
-       if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_REPL_UNLINK) &&
-           req->rq_reply_deadline > cfs_time_current_sec()) {
+       if (req->rq_reply_deadline > cfs_time_current_sec()) {
                spin_unlock(&req->rq_lock);
                return 1;
        }
+       if (req->rq_req_deadline > cfs_time_current_sec()) {
+               spin_unlock(&req->rq_lock);
+               return 1;
+       }
+
        rc = !req->rq_req_unlinked || !req->rq_reply_unlinked ||
             req->rq_receiving_reply;
        spin_unlock(&req->rq_lock);
@@ -2636,43 +2642,11 @@ void ptlrpc_pinger_ir_down(void);
 /** @} */
 int ptlrpc_pinger_suppress_pings(void);
 
-/* ptlrpc daemon bind policy */
-typedef enum {
-        /* all ptlrpcd threads are free mode */
-        PDB_POLICY_NONE          = 1,
-        /* all ptlrpcd threads are bound mode */
-        PDB_POLICY_FULL          = 2,
-        /* <free1 bound1> <free2 bound2> ... <freeN boundN> */
-        PDB_POLICY_PAIR          = 3,
-        /* <free1 bound1> <bound1 free2> ... <freeN boundN> <boundN free1>,
-         * means each ptlrpcd[X] has two partners: thread[X-1] and thread[X+1].
-         * If kernel supports NUMA, pthrpcd threads are binded and
-         * grouped by NUMA node */
-        PDB_POLICY_NEIGHBOR      = 4,
-} pdb_policy_t;
-
-/* ptlrpc daemon load policy
- * It is caller's duty to specify how to push the async RPC into some ptlrpcd
- * queue, but it is not enforced, affected by "ptlrpcd_bind_policy". If it is
- * "PDB_POLICY_FULL", then the RPC will be processed by the selected ptlrpcd,
- * Otherwise, the RPC may be processed by the selected ptlrpcd or its partner,
- * depends on which is scheduled firstly, to accelerate the RPC processing. */
-typedef enum {
-        /* on the same CPU core as the caller */
-        PDL_POLICY_SAME         = 1,
-        /* within the same CPU partition, but not the same core as the caller */
-        PDL_POLICY_LOCAL        = 2,
-        /* round-robin on all CPU cores, but not the same core as the caller */
-        PDL_POLICY_ROUND        = 3,
-        /* the specified CPU core is preferred, but not enforced */
-        PDL_POLICY_PREFERRED    = 4,
-} pdl_policy_t;
-
 /* ptlrpc/ptlrpcd.c */
 void ptlrpcd_stop(struct ptlrpcd_ctl *pc, int force);
 void ptlrpcd_free(struct ptlrpcd_ctl *pc);
 void ptlrpcd_wake(struct ptlrpc_request *req);
-void ptlrpcd_add_req(struct ptlrpc_request *req, pdl_policy_t policy, int idx);
+void ptlrpcd_add_req(struct ptlrpc_request *req);
 void ptlrpcd_add_rqset(struct ptlrpc_request_set *set);
 int ptlrpcd_addref(void);
 void ptlrpcd_decref(void);
@@ -2683,6 +2657,7 @@ void ptlrpcd_decref(void);
  * @{
  */
 const char* ll_opcode2str(__u32 opcode);
+const int ll_str2opcode(const char *ops);
 #ifdef CONFIG_PROC_FS
 void ptlrpc_lprocfs_register_obd(struct obd_device *obd);
 void ptlrpc_lprocfs_unregister_obd(struct obd_device *obd);
@@ -2704,7 +2679,6 @@ int llog_origin_handle_close(struct ptlrpc_request *req);
 
 /* ptlrpc/llog_client.c */
 extern struct llog_operations llog_client_ops;
-
 /** @} net */
 
 #endif