+ RETURN(rc);
+}
+
+/** Remove the request from the export list. */
+static void ptlrpc_hpreq_fini(struct ptlrpc_request *req)
+{
+ ENTRY;
+ if (req->rq_export && req->rq_ops) {
+ /* refresh lock timeout again so that client has more
+ * room to send lock cancel RPC. */
+ if (req->rq_ops->hpreq_fini)
+ req->rq_ops->hpreq_fini(req);
+
+ cfs_spin_lock_bh(&req->rq_export->exp_rpc_lock);
+ cfs_list_del_init(&req->rq_exp_list);
+ cfs_spin_unlock_bh(&req->rq_export->exp_rpc_lock);
+ }
+ EXIT;
+}
+
+/**
+ * Make the request a high priority one.
+ *
+ * All the high priority requests are queued in a separate FIFO
+ * ptlrpc_service_part::scp_hpreq_pending list which is parallel to
+ * ptlrpc_service_part::scp_req_pending list but has a higher priority
+ * for handling.
+ *
+ * \see ptlrpc_server_handle_request().
+ */
+static void ptlrpc_hpreq_reorder_nolock(struct ptlrpc_service_part *svcpt,
+ struct ptlrpc_request *req)
+{
+ ENTRY;
+
+ cfs_spin_lock(&req->rq_lock);
+ if (req->rq_hp == 0) {
+ int opc = lustre_msg_get_opc(req->rq_reqmsg);
+
+ /* Add to the high priority queue. */
+ cfs_list_move_tail(&req->rq_list, &svcpt->scp_hreq_pending);
+ req->rq_hp = 1;
+ if (opc != OBD_PING)
+ DEBUG_REQ(D_RPCTRACE, req, "high priority req");
+ }
+ cfs_spin_unlock(&req->rq_lock);
+ EXIT;
+}
+
+/**
+ * \see ptlrpc_hpreq_reorder_nolock
+ */
+void ptlrpc_hpreq_reorder(struct ptlrpc_request *req)
+{
+ struct ptlrpc_service_part *svcpt = req->rq_rqbd->rqbd_svcpt;
+ ENTRY;
+
+ cfs_spin_lock(&svcpt->scp_req_lock);
+ /* It may happen that the request is already taken for the processing
+ * but still in the export list, or the request is not in the request
+ * queue but in the export list already, do not add it into the
+ * HP list. */
+ if (!cfs_list_empty(&req->rq_list))
+ ptlrpc_hpreq_reorder_nolock(svcpt, req);
+ cfs_spin_unlock(&svcpt->scp_req_lock);
+ EXIT;
+}
+
+/** Check if the request is a high priority one. */
+static int ptlrpc_server_hpreq_check(struct ptlrpc_service *svc,
+ struct ptlrpc_request *req)
+{
+ ENTRY;
+
+ /* Check by request opc. */
+ if (OBD_PING == lustre_msg_get_opc(req->rq_reqmsg))
+ RETURN(1);
+
+ RETURN(ptlrpc_hpreq_init(svc, req));
+}
+
+/** Check if a request is a high priority one. */
+static int ptlrpc_server_request_add(struct ptlrpc_service_part *svcpt,
+ struct ptlrpc_request *req)
+{
+ int rc;
+ ENTRY;
+
+ rc = ptlrpc_server_hpreq_check(svcpt->scp_service, req);
+ if (rc < 0)
+ RETURN(rc);
+
+ cfs_spin_lock(&svcpt->scp_req_lock);
+
+ if (rc)
+ ptlrpc_hpreq_reorder_nolock(svcpt, req);
+ else
+ cfs_list_add_tail(&req->rq_list, &svcpt->scp_req_pending);
+
+ cfs_spin_unlock(&svcpt->scp_req_lock);
+
+ RETURN(0);
+}
+
+/**
+ * Allow to handle high priority request
+ * User can call it w/o any lock but need to hold
+ * ptlrpc_service_part::scp_req_lock to get reliable result
+ */
+static int ptlrpc_server_allow_high(struct ptlrpc_service_part *svcpt,
+ int force)
+{
+ if (force)
+ return 1;
+
+ if (svcpt->scp_nreqs_active >= svcpt->scp_nthrs_running - 1)
+ return 0;
+
+ return cfs_list_empty(&svcpt->scp_req_pending) ||
+ svcpt->scp_hreq_count < svcpt->scp_service->srv_hpreq_ratio;
+}
+
+static int ptlrpc_server_high_pending(struct ptlrpc_service_part *svcpt,
+ int force)
+{
+ return ptlrpc_server_allow_high(svcpt, force) &&
+ !cfs_list_empty(&svcpt->scp_hreq_pending);
+}
+
+/**
+ * Only allow normal priority requests on a service that has a high-priority
+ * queue if forced (i.e. cleanup), if there are other high priority requests
+ * already being processed (i.e. those threads can service more high-priority
+ * requests), or if there are enough idle threads that a later thread can do
+ * a high priority request.
+ * User can call it w/o any lock but need to hold
+ * ptlrpc_service_part::scp_req_lock to get reliable result
+ */
+static int ptlrpc_server_allow_normal(struct ptlrpc_service_part *svcpt,
+ int force)
+{
+#ifndef __KERNEL__
+ if (1) /* always allow to handle normal request for liblustre */
+ return 1;
+#endif
+ if (force ||
+ svcpt->scp_nreqs_active < svcpt->scp_nthrs_running - 2)
+ return 1;
+
+ if (svcpt->scp_nreqs_active >= svcpt->scp_nthrs_running - 1)
+ return 0;
+
+ return svcpt->scp_nhreqs_active > 0 ||
+ svcpt->scp_service->srv_ops.so_hpreq_handler == NULL;
+}
+
+static int ptlrpc_server_normal_pending(struct ptlrpc_service_part *svcpt,
+ int force)
+{
+ return ptlrpc_server_allow_normal(svcpt, force) &&
+ !cfs_list_empty(&svcpt->scp_req_pending);
+}
+
+/**
+ * Returns true if there are requests available in incoming
+ * request queue for processing and it is allowed to fetch them.
+ * User can call it w/o any lock but need to hold ptlrpc_service::scp_req_lock
+ * to get reliable result
+ * \see ptlrpc_server_allow_normal
+ * \see ptlrpc_server_allow high
+ */
+static inline int
+ptlrpc_server_request_pending(struct ptlrpc_service_part *svcpt, int force)
+{
+ return ptlrpc_server_high_pending(svcpt, force) ||
+ ptlrpc_server_normal_pending(svcpt, force);
+}
+
+/**
+ * Fetch a request for processing from queue of unprocessed requests.
+ * Favors high-priority requests.
+ * Returns a pointer to fetched request.
+ */
+static struct ptlrpc_request *
+ptlrpc_server_request_get(struct ptlrpc_service_part *svcpt, int force)
+{
+ struct ptlrpc_request *req;
+ ENTRY;
+
+ if (ptlrpc_server_high_pending(svcpt, force)) {
+ req = cfs_list_entry(svcpt->scp_hreq_pending.next,
+ struct ptlrpc_request, rq_list);
+ svcpt->scp_hreq_count++;
+ RETURN(req);
+ }
+
+ if (ptlrpc_server_normal_pending(svcpt, force)) {
+ req = cfs_list_entry(svcpt->scp_req_pending.next,
+ struct ptlrpc_request, rq_list);
+ svcpt->scp_hreq_count = 0;
+ RETURN(req);
+ }
+ RETURN(NULL);
+}
+
+/**
+ * Handle freshly incoming reqs, add to timed early reply list,
+ * pass on to regular request queue.
+ * All incoming requests pass through here before getting into
+ * ptlrpc_server_handle_req later on.
+ */
+static int
+ptlrpc_server_handle_req_in(struct ptlrpc_service_part *svcpt)
+{
+ struct ptlrpc_service *svc = svcpt->scp_service;
+ struct ptlrpc_request *req;
+ __u32 deadline;
+ int rc;
+ ENTRY;
+
+ cfs_spin_lock(&svcpt->scp_lock);
+ if (cfs_list_empty(&svcpt->scp_req_incoming)) {
+ cfs_spin_unlock(&svcpt->scp_lock);
+ RETURN(0);
+ }
+
+ req = cfs_list_entry(svcpt->scp_req_incoming.next,
+ struct ptlrpc_request, rq_list);
+ cfs_list_del_init(&req->rq_list);
+ svcpt->scp_nreqs_incoming--;
+ /* Consider this still a "queued" request as far as stats are
+ * concerned */
+ cfs_spin_unlock(&svcpt->scp_lock);