- srpc_event_t *rpcev = ev->md.user_ptr;
- srpc_client_rpc_t *crpc;
- srpc_server_rpc_t *srpc;
- srpc_buffer_t *buffer;
- srpc_service_t *sv;
- srpc_msg_t *msg;
- srpc_msg_type_t type;
-
- LASSERT (!cfs_in_interrupt());
-
- if (ev->status != 0) {
- cfs_spin_lock(&srpc_data.rpc_glock);
- srpc_data.rpc_counters.errors++;
- cfs_spin_unlock(&srpc_data.rpc_glock);
- }
-
- rpcev->ev_lnet = ev->type;
-
- switch (rpcev->ev_type) {
- default:
- CERROR("Unknown event: status %d, type %d, lnet %d\n",
- rpcev->ev_status, rpcev->ev_type, rpcev->ev_lnet);
- LBUG ();
- case SRPC_REQUEST_SENT:
- if (ev->status == 0 && ev->type != LNET_EVENT_UNLINK) {
- cfs_spin_lock(&srpc_data.rpc_glock);
- srpc_data.rpc_counters.rpcs_sent++;
- cfs_spin_unlock(&srpc_data.rpc_glock);
- }
- case SRPC_REPLY_RCVD:
- case SRPC_BULK_REQ_RCVD:
- crpc = rpcev->ev_data;
-
- if (rpcev != &crpc->crpc_reqstev &&
- rpcev != &crpc->crpc_replyev &&
- rpcev != &crpc->crpc_bulkev) {
- CERROR("rpcev %p, crpc %p, reqstev %p, replyev %p, bulkev %p\n",
- rpcev, crpc, &crpc->crpc_reqstev,
- &crpc->crpc_replyev, &crpc->crpc_bulkev);
- CERROR("Bad event: status %d, type %d, lnet %d\n",
- rpcev->ev_status, rpcev->ev_type, rpcev->ev_lnet);
- LBUG ();
- }
-
- cfs_spin_lock(&crpc->crpc_lock);
-
- LASSERT (rpcev->ev_fired == 0);
- rpcev->ev_fired = 1;
- rpcev->ev_status = (ev->type == LNET_EVENT_UNLINK) ?
- -EINTR : ev->status;
- swi_schedule_workitem(&crpc->crpc_wi);
-
- cfs_spin_unlock(&crpc->crpc_lock);
- break;
-
- case SRPC_REQUEST_RCVD:
- sv = rpcev->ev_data;
-
- LASSERT (rpcev == &sv->sv_ev);
-
- cfs_spin_lock(&sv->sv_lock);
-
- LASSERT (ev->unlinked);
- LASSERT (ev->type == LNET_EVENT_PUT ||
- ev->type == LNET_EVENT_UNLINK);
- LASSERT (ev->type != LNET_EVENT_UNLINK ||
- sv->sv_shuttingdown);
-
- buffer = container_of(ev->md.start, srpc_buffer_t, buf_msg);
- buffer->buf_peer = ev->initiator;
- buffer->buf_self = ev->target.nid;
-
- sv->sv_nposted_msg--;
- LASSERT (sv->sv_nposted_msg >= 0);
-
- if (sv->sv_shuttingdown) {
- /* Leave buffer on sv->sv_posted_msgq since
- * srpc_finish_service needs to traverse it. */
- cfs_spin_unlock(&sv->sv_lock);
- break;
- }
-
- cfs_list_del(&buffer->buf_list); /* from sv->sv_posted_msgq */
- msg = &buffer->buf_msg;
- type = srpc_service2request(sv->sv_id);
-
- if (ev->status != 0 || ev->mlength != sizeof(*msg) ||
- (msg->msg_type != type &&
- msg->msg_type != __swab32(type)) ||
- (msg->msg_magic != SRPC_MSG_MAGIC &&
- msg->msg_magic != __swab32(SRPC_MSG_MAGIC))) {
- CERROR ("Dropping RPC (%s) from %s: "
- "status %d mlength %d type %u magic %u.\n",
- sv->sv_name, libcfs_id2str(ev->initiator),
- ev->status, ev->mlength,
- msg->msg_type, msg->msg_magic);
-
- /* NB can't call srpc_service_recycle_buffer here since
- * it may call LNetM[DE]Attach. The invalid magic tells
- * srpc_handle_rpc to drop this RPC */
- msg->msg_magic = 0;
- }
-
- if (!cfs_list_empty(&sv->sv_free_rpcq)) {
- srpc = cfs_list_entry(sv->sv_free_rpcq.next,
- srpc_server_rpc_t, srpc_list);
- cfs_list_del(&srpc->srpc_list);
-
- srpc_init_server_rpc(srpc, sv, buffer);
- cfs_list_add_tail(&srpc->srpc_list,
- &sv->sv_active_rpcq);
- srpc_schedule_server_rpc(srpc);
- } else {
- cfs_list_add_tail(&buffer->buf_list,
- &sv->sv_blocked_msgq);
- }
-
- cfs_spin_unlock(&sv->sv_lock);
-
- cfs_spin_lock(&srpc_data.rpc_glock);
- srpc_data.rpc_counters.rpcs_rcvd++;
- cfs_spin_unlock(&srpc_data.rpc_glock);
- break;
-
- case SRPC_BULK_GET_RPLD:
- LASSERT (ev->type == LNET_EVENT_SEND ||
- ev->type == LNET_EVENT_REPLY ||
- ev->type == LNET_EVENT_UNLINK);
-
- if (!ev->unlinked)
- break; /* wait for final event */
-
- case SRPC_BULK_PUT_SENT:
- if (ev->status == 0 && ev->type != LNET_EVENT_UNLINK) {
- cfs_spin_lock(&srpc_data.rpc_glock);
-
- if (rpcev->ev_type == SRPC_BULK_GET_RPLD)
- srpc_data.rpc_counters.bulk_get += ev->mlength;
- else
- srpc_data.rpc_counters.bulk_put += ev->mlength;
-
- cfs_spin_unlock(&srpc_data.rpc_glock);
- }
- case SRPC_REPLY_SENT:
- srpc = rpcev->ev_data;
- sv = srpc->srpc_service;
-
- LASSERT (rpcev == &srpc->srpc_ev);
-
- cfs_spin_lock(&sv->sv_lock);
-
- rpcev->ev_fired = 1;
- rpcev->ev_status = (ev->type == LNET_EVENT_UNLINK) ?
- -EINTR : ev->status;
- srpc_schedule_server_rpc(srpc);
-
- cfs_spin_unlock(&sv->sv_lock);
- break;
- }
-
- return;
+ struct srpc_service_cd *scd;
+ struct srpc_event *rpcev = ev->md.user_ptr;
+ struct srpc_client_rpc *crpc;
+ struct srpc_server_rpc *srpc;
+ struct srpc_buffer *buffer;
+ struct srpc_service *sv;
+ struct srpc_msg *msg;
+ enum srpc_msg_type type;
+
+ LASSERT(!in_interrupt());
+
+ if (ev->status != 0) {
+ __u32 errors;
+
+ spin_lock(&srpc_data.rpc_glock);
+ if (ev->status != -ECANCELED) /* cancellation is not error */
+ srpc_data.rpc_counters.errors++;
+ errors = srpc_data.rpc_counters.errors;
+ spin_unlock(&srpc_data.rpc_glock);
+
+ CNETERR("LNet event status %d type %d, RPC errors %u\n",
+ ev->status, ev->type, errors);
+ }
+
+ rpcev->ev_lnet = ev->type;
+
+ switch (rpcev->ev_type) {
+ default:
+ CERROR("Unknown event: status %d, type %d, lnet %d\n",
+ rpcev->ev_status, rpcev->ev_type, rpcev->ev_lnet);
+ LBUG();
+ /* fallthrough */
+ case SRPC_REQUEST_SENT:
+ if (ev->status == 0 && ev->type != LNET_EVENT_UNLINK) {
+ spin_lock(&srpc_data.rpc_glock);
+ srpc_data.rpc_counters.rpcs_sent++;
+ spin_unlock(&srpc_data.rpc_glock);
+ }
+ /* fallthrough */
+ case SRPC_REPLY_RCVD:
+ case SRPC_BULK_REQ_RCVD:
+ crpc = rpcev->ev_data;
+
+ if (rpcev != &crpc->crpc_reqstev &&
+ rpcev != &crpc->crpc_replyev &&
+ rpcev != &crpc->crpc_bulkev) {
+ CERROR("rpcev %p, crpc %p, reqstev %p, replyev %p, bulkev %p\n",
+ rpcev, crpc, &crpc->crpc_reqstev,
+ &crpc->crpc_replyev, &crpc->crpc_bulkev);
+ CERROR("Bad event: status %d, type %d, lnet %d\n",
+ rpcev->ev_status, rpcev->ev_type,
+ rpcev->ev_lnet);
+ LBUG();
+ }
+
+ spin_lock(&crpc->crpc_lock);
+
+ LASSERT(rpcev->ev_fired == 0);
+ rpcev->ev_fired = 1;
+ rpcev->ev_status = (ev->type == LNET_EVENT_UNLINK) ?
+ -EINTR : ev->status;
+ swi_schedule_workitem(&crpc->crpc_wi);
+
+ spin_unlock(&crpc->crpc_lock);
+ break;
+
+ case SRPC_REQUEST_RCVD:
+ scd = rpcev->ev_data;
+ sv = scd->scd_svc;
+
+ LASSERT(rpcev == &scd->scd_ev);
+
+ spin_lock(&scd->scd_lock);
+
+ LASSERT(ev->unlinked);
+ LASSERT(ev->type == LNET_EVENT_PUT ||
+ ev->type == LNET_EVENT_UNLINK);
+ LASSERT(ev->type != LNET_EVENT_UNLINK ||
+ sv->sv_shuttingdown);
+
+ buffer = container_of(ev->md.start, struct srpc_buffer,
+ buf_msg);
+ buffer->buf_peer = ev->source;
+ buffer->buf_self = ev->target.nid;
+
+ LASSERT(scd->scd_buf_nposted > 0);
+ scd->scd_buf_nposted--;
+
+ if (sv->sv_shuttingdown) {
+ /* Leave buffer on scd->scd_buf_nposted since
+ * srpc_finish_service needs to traverse it.
+ */
+ spin_unlock(&scd->scd_lock);
+ break;
+ }
+
+ if (scd->scd_buf_err_stamp != 0 &&
+ scd->scd_buf_err_stamp < ktime_get_real_seconds()) {
+ /* re-enable adding buffer */
+ scd->scd_buf_err_stamp = 0;
+ scd->scd_buf_err = 0;
+ }
+
+ if (scd->scd_buf_err == 0 && /* adding buffer is enabled */
+ scd->scd_buf_adjust == 0 &&
+ scd->scd_buf_nposted < scd->scd_buf_low) {
+ scd->scd_buf_adjust = MAX(scd->scd_buf_total / 2,
+ SFW_TEST_WI_MIN);
+ swi_schedule_workitem(&scd->scd_buf_wi);
+ }
+
+ list_del(&buffer->buf_list); /* from scd->scd_buf_posted */
+ msg = &buffer->buf_msg;
+ type = srpc_service2request(sv->sv_id);
+
+ if (ev->status != 0 || ev->mlength != sizeof(*msg) ||
+ (msg->msg_type != type &&
+ msg->msg_type != __swab32(type)) ||
+ (msg->msg_magic != SRPC_MSG_MAGIC &&
+ msg->msg_magic != __swab32(SRPC_MSG_MAGIC))) {
+ CERROR("Dropping RPC (%s) from %s: status %d mlength %d type %u magic %u.\n",
+ sv->sv_name, libcfs_id2str(ev->initiator),
+ ev->status, ev->mlength,
+ msg->msg_type, msg->msg_magic);
+
+ /* NB can't call srpc_service_recycle_buffer here since
+ * it may call LNetM[DE]Attach. The invalid magic tells
+ * srpc_handle_rpc to drop this RPC
+ */
+ msg->msg_magic = 0;
+ }
+
+ if (!list_empty(&scd->scd_rpc_free)) {
+ srpc = list_entry(scd->scd_rpc_free.next,
+ struct srpc_server_rpc,
+ srpc_list);
+ list_del(&srpc->srpc_list);
+
+ srpc_init_server_rpc(srpc, scd, buffer);
+ list_add_tail(&srpc->srpc_list,
+ &scd->scd_rpc_active);
+ swi_schedule_workitem(&srpc->srpc_wi);
+ } else {
+ list_add_tail(&buffer->buf_list,
+ &scd->scd_buf_blocked);
+ }
+
+ spin_unlock(&scd->scd_lock);
+
+ spin_lock(&srpc_data.rpc_glock);
+ srpc_data.rpc_counters.rpcs_rcvd++;
+ spin_unlock(&srpc_data.rpc_glock);
+ break;
+
+ case SRPC_BULK_GET_RPLD:
+ LASSERT(ev->type == LNET_EVENT_SEND ||
+ ev->type == LNET_EVENT_REPLY ||
+ ev->type == LNET_EVENT_UNLINK);
+
+ if (!ev->unlinked)
+ break; /* wait for final event */
+ /* fallthrough */
+ case SRPC_BULK_PUT_SENT:
+ if (ev->status == 0 && ev->type != LNET_EVENT_UNLINK) {
+ spin_lock(&srpc_data.rpc_glock);
+
+ if (rpcev->ev_type == SRPC_BULK_GET_RPLD)
+ srpc_data.rpc_counters.bulk_get += ev->mlength;
+ else
+ srpc_data.rpc_counters.bulk_put += ev->mlength;
+
+ spin_unlock(&srpc_data.rpc_glock);
+ }
+ /* fallthrough */
+ case SRPC_REPLY_SENT:
+ srpc = rpcev->ev_data;
+ scd = srpc->srpc_scd;
+
+ LASSERT(rpcev == &srpc->srpc_ev);
+
+ spin_lock(&scd->scd_lock);
+
+ rpcev->ev_fired = 1;
+ rpcev->ev_status = (ev->type == LNET_EVENT_UNLINK) ?
+ -EINTR : ev->status;
+ swi_schedule_workitem(&srpc->srpc_wi);
+
+ spin_unlock(&scd->scd_lock);
+ break;
+ }