+/** Checking routines for recovery */
+static int check_for_recovery_ready(struct lu_target *lut)
+{
+ struct obd_device *obd = lut->lut_obd;
+ unsigned int clnts = atomic_read(&obd->obd_connected_clients);
+
+ CDEBUG(D_HA, "connected %d stale %d max_recoverable_clients %d"
+ " abort %d expired %d\n", clnts, obd->obd_stale_clients,
+ obd->obd_max_recoverable_clients, obd->obd_abort_recovery,
+ obd->obd_recovery_expired);
+
+ if (!obd->obd_abort_recovery && !obd->obd_recovery_expired) {
+ LASSERT(clnts <= obd->obd_max_recoverable_clients);
+ if (clnts + obd->obd_stale_clients <
+ obd->obd_max_recoverable_clients)
+ return 0;
+ }
+
+ if (lut->lut_tdtd != NULL) {
+ if (!lut->lut_tdtd->tdtd_replay_ready &&
+ !obd->obd_abort_recovery) {
+ /* Let's extend recovery timer, in case the recovery
+ * timer expired, and some clients got evicted */
+ extend_recovery_timer(obd, obd->obd_recovery_timeout,
+ true);
+ CDEBUG(D_HA, "%s update recovery is not ready, extend recovery %llu\n",
+ obd->obd_name, obd->obd_recovery_timeout);
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+enum {
+ REQUEST_RECOVERY = 1,
+ UPDATE_RECOVERY = 2,
+};
+
+static __u64 get_next_replay_req_transno(struct obd_device *obd)
+{
+ __u64 transno = 0;
+
+ if (!list_empty(&obd->obd_req_replay_queue)) {
+ struct ptlrpc_request *req;
+
+ req = list_entry(obd->obd_req_replay_queue.next,
+ struct ptlrpc_request, rq_list);
+ transno = lustre_msg_get_transno(req->rq_reqmsg);
+ }
+
+ return transno;
+}
+
+static __u64 get_next_transno(struct lu_target *lut, int *type)
+{
+ struct obd_device *obd = lut->lut_obd;
+ struct target_distribute_txn_data *tdtd = lut->lut_tdtd;
+ __u64 transno = 0;
+ __u64 update_transno;
+ ENTRY;
+
+ transno = get_next_replay_req_transno(obd);
+ if (type != NULL)
+ *type = REQUEST_RECOVERY;
+
+ if (tdtd == NULL)
+ RETURN(transno);
+
+ update_transno = distribute_txn_get_next_transno(tdtd);
+ if (transno == 0 || (transno >= update_transno &&
+ update_transno != 0)) {
+ transno = update_transno;
+ if (type != NULL)
+ *type = UPDATE_RECOVERY;
+ }
+
+ RETURN(transno);
+}
+
+/**
+ * drop duplicate replay request
+ *
+ * Because the operation has been replayed by update recovery, the request
+ * with the same transno will be dropped and also notify the client to send
+ * next replay request.
+ *
+ * \param[in] env execution environment
+ * \param[in] obd failover obd device
+ * \param[in] req request to be dropped
+ */
+static void drop_duplicate_replay_req(struct lu_env *env,
+ struct obd_device *obd,
+ struct ptlrpc_request *req)
+{
+ DEBUG_REQ(D_HA, req, "remove t%lld from %s because of duplicate"
+ " update records are found.\n",
+ lustre_msg_get_transno(req->rq_reqmsg),
+ libcfs_nid2str(req->rq_peer.nid));
+
+ /* Right now, only for MDS reint operation update replay and
+ * normal request replay can have the same transno */
+ if (lustre_msg_get_opc(req->rq_reqmsg) == MDS_REINT) {
+ req_capsule_set(&req->rq_pill, &RQF_MDS_REINT);
+ req->rq_status = req_capsule_server_pack(&req->rq_pill);
+ if (likely(req->rq_export))
+ target_committed_to_req(req);
+ lustre_msg_set_transno(req->rq_repmsg, req->rq_transno);
+ target_send_reply(req, req->rq_status, 0);
+ } else {
+ DEBUG_REQ(D_ERROR, req, "wrong opc" "from %s\n",
+ libcfs_nid2str(req->rq_peer.nid));
+ }
+ target_exp_dequeue_req_replay(req);
+ target_request_copy_put(req);
+ obd->obd_replayed_requests++;
+}
+
+static void replay_request_or_update(struct lu_env *env,
+ struct lu_target *lut,
+ struct target_recovery_data *trd,
+ struct ptlrpc_thread *thread)
+{
+ struct obd_device *obd = lut->lut_obd;
+ struct ptlrpc_request *req = NULL;
+ int type;
+ __u64 transno;
+ ENTRY;
+
+ CDEBUG(D_HA, "Waiting for transno %lld\n",
+ obd->obd_next_recovery_transno);
+
+ /* Replay all of request and update by transno */
+ do {
+ struct target_distribute_txn_data *tdtd = lut->lut_tdtd;
+
+ CFS_FAIL_TIMEOUT(OBD_FAIL_TGT_REPLAY_DELAY2, cfs_fail_val);
+
+ /** It is needed to extend recovery window above
+ * recovery_time_soft. Extending is possible only in the
+ * end of recovery window (see more details in
+ * handle_recovery_req()).
+ */
+ CFS_FAIL_TIMEOUT_MS(OBD_FAIL_TGT_REPLAY_DELAY, 300);
+
+ if (target_recovery_overseer(lut, check_for_next_transno,
+ exp_req_replay_healthy_or_from_mdt)) {
+ abort_req_replay_queue(obd);
+ abort_lock_replay_queue(obd);
+ goto abort;
+ }
+
+ spin_lock(&obd->obd_recovery_task_lock);
+ transno = get_next_transno(lut, &type);
+ if (type == REQUEST_RECOVERY && transno != 0) {
+ /* Drop replay request from client side, if the
+ * replay has been executed by update with the
+ * same transno */
+ req = list_entry(obd->obd_req_replay_queue.next,
+ struct ptlrpc_request, rq_list);
+
+ list_del_init(&req->rq_list);
+ obd->obd_requests_queued_for_recovery--;
+ spin_unlock(&obd->obd_recovery_task_lock);
+
+ /* Let's check if the request has been redone by
+ * update replay */
+ if (is_req_replayed_by_update(req)) {
+ struct distribute_txn_replay_req *dtrq;
+
+ dtrq = distribute_txn_lookup_finish_list(tdtd,
+ req->rq_xid);
+ LASSERT(dtrq != NULL);
+ spin_lock(&tdtd->tdtd_replay_list_lock);
+ list_del_init(&dtrq->dtrq_list);
+ spin_unlock(&tdtd->tdtd_replay_list_lock);
+ dtrq_destroy(dtrq);
+
+ drop_duplicate_replay_req(env, obd, req);
+
+ continue;
+ }
+
+ LASSERT(trd->trd_processing_task == current_pid());
+ DEBUG_REQ(D_HA, req, "processing t%lld from %s",
+ lustre_msg_get_transno(req->rq_reqmsg),
+ libcfs_nid2str(req->rq_peer.nid));
+
+ handle_recovery_req(thread, req,
+ trd->trd_recovery_handler);
+ /**
+ * bz18031: increase next_recovery_transno before
+ * target_request_copy_put() will drop exp_rpc reference
+ */
+ spin_lock(&obd->obd_recovery_task_lock);
+ obd->obd_next_recovery_transno++;
+ spin_unlock(&obd->obd_recovery_task_lock);
+ target_exp_dequeue_req_replay(req);
+ target_request_copy_put(req);
+ obd->obd_replayed_requests++;
+ } else if (type == UPDATE_RECOVERY && transno != 0) {
+ struct distribute_txn_replay_req *dtrq;
+ int rc;
+
+ spin_unlock(&obd->obd_recovery_task_lock);
+
+ LASSERT(tdtd != NULL);
+ dtrq = distribute_txn_get_next_req(tdtd);
+ lu_context_enter(&thread->t_env->le_ctx);
+ rc = tdtd->tdtd_replay_handler(env, tdtd, dtrq);
+ lu_context_exit(&thread->t_env->le_ctx);
+ extend_recovery_timer(obd, obd_timeout, true);
+
+ if (rc == 0 && dtrq->dtrq_xid != 0) {
+ CDEBUG(D_HA, "Move x%llu t%llu"
+ " to finish list\n", dtrq->dtrq_xid,
+ dtrq->dtrq_master_transno);
+
+ /* Add it to the replay finish list */
+ spin_lock(&tdtd->tdtd_replay_list_lock);
+ list_add(&dtrq->dtrq_list,
+ &tdtd->tdtd_replay_finish_list);
+ spin_unlock(&tdtd->tdtd_replay_list_lock);
+
+ spin_lock(&obd->obd_recovery_task_lock);
+ if (transno == obd->obd_next_recovery_transno)
+ obd->obd_next_recovery_transno++;
+ else if (transno >
+ obd->obd_next_recovery_transno)
+ obd->obd_next_recovery_transno =
+ transno + 1;
+ spin_unlock(&obd->obd_recovery_task_lock);
+ } else {
+ dtrq_destroy(dtrq);
+ }
+ } else {
+ spin_unlock(&obd->obd_recovery_task_lock);
+abort:
+ LASSERT(list_empty(&obd->obd_req_replay_queue));
+ LASSERT(atomic_read(&obd->obd_req_replay_clients) == 0);
+ /** evict exports failed VBR */
+ class_disconnect_stale_exports(obd, exp_vbr_healthy);
+ break;
+ }
+ } while (1);
+}
+