4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.gnu.org/licenses/gpl-2.0.html
23 * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Use is subject to license terms.
26 * Copyright (c) 2011, 2016, Intel Corporation.
29 * This file is part of Lustre, http://www.lustre.org/
31 /** \defgroup obd_import PtlRPC import definitions
32 * Imports are client-side representation of remote obd target.
40 /** \defgroup export export
44 #include <linux/atomic.h>
45 #include <linux/list.h>
46 #include <linux/mutex.h>
47 #include <linux/refcount.h>
48 #include <linux/spinlock.h>
49 #include <linux/time.h>
50 #include <linux/types.h>
51 #include <linux/workqueue.h>
52 #include <libcfs/libcfs.h>
53 #include <uapi/linux/lustre/lustre_idl.h>
56 * Adaptive Timeout stuff
60 #define D_ADAPTTO D_OTHER
61 #define AT_BINS 4 /* "bin" means "N seconds of history" */
62 #define AT_FLG_NOHIST 0x1 /* use last reported value only */
64 struct adaptive_timeout {
65 time64_t at_binstart; /* bin start time */
66 unsigned int at_hist[AT_BINS]; /* timeout history bins */
67 unsigned int at_flags;
68 timeout_t at_current_timeout; /* current timeout value */
69 timeout_t at_worst_timeout_ever; /* worst-ever timeout delta
72 time64_t at_worst_timestamp; /* worst-ever timeout
78 enum lustre_at_flags {
83 struct ptlrpc_at_array {
84 struct list_head *paa_reqs_array; /** array to hold requests */
85 __u32 paa_size; /** the size of array */
86 __u32 paa_count; /** the total count of reqs */
87 time64_t paa_deadline; /** the earliest deadline of reqs */
88 __u32 *paa_reqs_count; /** the count of reqs in each entry */
91 #define IMP_AT_MAX_PORTALS 8
93 int iat_portal[IMP_AT_MAX_PORTALS];
94 struct adaptive_timeout iat_net_latency;
95 struct adaptive_timeout iat_service_estimate[IMP_AT_MAX_PORTALS];
101 /** Possible import states */
102 enum lustre_imp_state {
103 LUSTRE_IMP_CLOSED = 1,
105 LUSTRE_IMP_DISCON = 3,
106 LUSTRE_IMP_CONNECTING = 4,
107 LUSTRE_IMP_REPLAY = 5,
108 LUSTRE_IMP_REPLAY_LOCKS = 6,
109 LUSTRE_IMP_REPLAY_WAIT = 7,
110 LUSTRE_IMP_RECOVER = 8,
112 LUSTRE_IMP_EVICTED = 10,
113 LUSTRE_IMP_IDLE = 11,
117 /** Returns test string representation of numeric import state \a state */
118 static inline const char *ptlrpc_import_state_name(enum lustre_imp_state state)
120 static const char * const import_state_names[] = {
121 "<UNKNOWN>", "CLOSED", "NEW", "DISCONN",
122 "CONNECTING", "REPLAY", "REPLAY_LOCKS", "REPLAY_WAIT",
123 "RECOVER", "FULL", "EVICTED", "IDLE",
126 LASSERT(state < LUSTRE_IMP_LAST);
127 return import_state_names[state];
131 * List of import event types
133 enum obd_import_event {
134 IMP_EVENT_DISCON = 0x808001,
135 IMP_EVENT_INACTIVE = 0x808002,
136 IMP_EVENT_INVALIDATE = 0x808003,
137 IMP_EVENT_ACTIVE = 0x808004,
138 IMP_EVENT_OCD = 0x808005,
139 IMP_EVENT_DEACTIVATE = 0x808006,
140 IMP_EVENT_ACTIVATE = 0x808007,
144 * Definition of import connection structure
146 struct obd_import_conn {
147 /** Item for linking connections together */
148 struct list_head oic_item;
149 /** Pointer to actual PortalRPC connection */
150 struct ptlrpc_connection *oic_conn;
151 /** uuid of remote side */
152 struct obd_uuid oic_uuid;
154 * Time (64 bit seconds) of last connection attempt on this connection
156 time64_t oic_last_attempt;
157 unsigned int oic_attempts;
158 unsigned int oic_replied;
163 #define IMP_STATE_HIST_LEN 16
164 struct import_state_hist {
165 enum lustre_imp_state ish_state;
170 * Defintion of PortalRPC import structure.
171 * Imports are representing client-side view to remote target.
174 /** Reference counter */
175 refcount_t imp_refcount;
176 struct lustre_handle imp_dlm_handle; /* client's ldlm export */
177 /** Currently active connection */
178 struct ptlrpc_connection *imp_connection;
179 /** PortalRPC client structure for this import */
180 struct ptlrpc_client *imp_client;
181 /** List element for linking into pinger chain */
182 struct list_head imp_pinger_chain;
183 /** work struct for destruction of import */
184 struct work_struct imp_zombie_work;
187 * Lists of requests that are retained for replay, waiting for a reply,
188 * or waiting for recovery to complete, respectively.
191 struct list_head imp_replay_list;
192 struct list_head imp_sending_list;
193 struct list_head imp_delayed_list;
197 * List of requests that are retained for committed open replay. Once
198 * open is committed, open replay request will be moved from the
199 * imp_replay_list into the imp_committed_list.
200 * The imp_replay_cursor is for accelerating searching during replay.
203 struct list_head imp_committed_list;
204 struct list_head *imp_replay_cursor;
207 /** List of not replied requests */
208 struct list_head imp_unreplied_list;
209 /** XID below which we know all replies have been received */
210 __u64 imp_known_replied_xid;
211 /** highest XID for which we have received a reply */
212 __u64 imp_highest_replied_xid;
214 /** obd device for this import */
215 struct obd_device *imp_obd;
218 * some seciruty-related fields
221 struct ptlrpc_sec *imp_sec;
222 rwlock_t imp_sec_lock;
223 time64_t imp_sec_expire;
224 pid_t imp_sec_refpid;
227 /** Wait queue for those who need to wait for recovery completion */
228 wait_queue_head_t imp_recovery_waitq;
230 /** Number of requests allocated */
232 /** Number of requests currently in-flight */
233 atomic_t imp_inflight;
234 /** Number of requests currently unregistering */
235 atomic_t imp_unregistering;
236 /** Number of replay requests inflight */
237 atomic_t imp_replay_inflight;
238 /** In-flight replays rate control */
239 wait_queue_head_t imp_replay_waitq;
241 /** Number of currently happening import invalidations */
242 atomic_t imp_inval_count;
243 /** Numbner of request timeouts */
244 atomic_t imp_timeouts;
245 /** Current import state */
246 enum lustre_imp_state imp_state;
247 /** Last replay state */
248 enum lustre_imp_state imp_replay_state;
249 /** History of import states */
250 struct import_state_hist imp_state_hist[IMP_STATE_HIST_LEN];
251 int imp_state_hist_idx;
252 /** Current import generation. Incremented on every reconnect */
254 /** Idle connection initiated at this generation */
255 int imp_initiated_at;
256 /** Incremented every time we send reconnection request */
259 * \see ptlrpc_free_committed remembers imp_generation value here
260 * after a check to save on unnecessary replay list iterations
262 int imp_last_generation_checked;
263 /** Last tranno we replayed */
264 __u64 imp_last_replay_transno;
265 /** Last transno committed on remote side */
266 __u64 imp_peer_committed_transno;
268 * \see ptlrpc_free_committed remembers last_transno since its last
269 * check here and if last_transno did not change since last run of
270 * ptlrpc_free_committed and import generation is the same, we can
271 * skip looking for requests to remove from replay list as optimisation
273 __u64 imp_last_transno_checked;
275 * Remote export handle. This is how remote side knows what export
276 * we are talking to. Filled from response to connect request
278 struct lustre_handle imp_remote_handle;
279 /** When to perform next ping. time in jiffies. */
280 time64_t imp_next_ping;
281 /** When we last successfully connected. time in 64bit jiffies */
282 time64_t imp_last_success_conn;
284 /** List of all possible connection for import. */
285 struct list_head imp_conn_list;
287 * Current connection. \a imp_connection is imp_conn_current->oic_conn
289 struct obd_import_conn *imp_conn_current;
291 /** Protects flags, level, generation, conn_cnt, *_list */
295 * A "sentinel" value used to check if there are other threads
296 * waiting on the imp_lock.
298 atomic_t imp_waiting;
301 unsigned long imp_invalid:1, /* evicted */
302 /* administratively disabled */
304 /* try to recover the import */
306 /* don't run recovery (timeout instead) */
308 /* use 1/2 timeout on MDS' OSCs */
309 imp_server_timeout:1,
310 /* VBR: imp in delayed recovery */
311 imp_delayed_recovery:1,
312 /* recovery by versions was failed */
314 /* force an immidiate ping */
316 /* force a scheduled ping */
317 imp_force_next_verify:1,
320 /* resend for replay */
322 /* disable normal recovery, for test only. */
323 imp_no_pinger_recover:1,
324 /* import must be reconnected instead of
325 * chouse new connection */
326 imp_force_reconnect:1,
327 /* import has tried to connect with server */
329 /* connected but not FULL yet */
331 /* grant shrink disabled */
332 imp_grant_shrink_disabled:1,
333 /* to supress LCONSOLE() at conn.restore */
335 imp_no_cached_data:1;
337 u32 imp_idle_timeout;
339 struct obd_connect_data imp_connect_data;
340 __u64 imp_connect_flags_orig;
341 __u64 imp_connect_flags2_orig;
342 int imp_connect_error;
344 enum lustre_msg_magic imp_msg_magic;
345 /* adjusted based on server capability */
346 enum lustre_msghdr imp_msghdr_flags;
348 /* adaptive timeout data */
349 struct imp_at imp_at;
350 time64_t imp_last_reply_time; /* for health check */
351 time64_t imp_setup_time;
352 __u32 imp_conn_restricted_net;
355 /* import.c : adaptive timeout handling.
357 * Lustre tracks how long RPCs take to complete. This information is reported
358 * back to clients who utilize the information to estimate the time needed
359 * for future requests and set appropriate RPC timeouts. Minimum and maximum
360 * service times can be configured via the at_min and at_max kernel module
361 * parameters, respectively.
363 * Since this information is transmitted between nodes the timeouts are in
364 * seconds not jiffies which can vary from node to node. To avoid confusion
365 * the timeout is handled in timeout_t (s32) instead of time64_t or
368 static inline timeout_t at_est2timeout(timeout_t timeout)
370 /* add an arbitrary minimum: 125% +5 sec */
371 return timeout + (timeout >> 2) + 5;
374 static inline timeout_t at_timeout2est(timeout_t timeout)
376 /* restore estimate value from timeout: e=4/5(t-5) */
377 LASSERT(timeout > 0);
378 return max((timeout << 2) / 5, 5) - 4;
381 static inline void at_reset_nolock(struct adaptive_timeout *at,
384 at->at_current_timeout = timeout;
385 at->at_worst_timeout_ever = timeout;
386 at->at_worst_timestamp = ktime_get_real_seconds();
389 static inline void at_reset(struct adaptive_timeout *at, timeout_t timeout)
391 spin_lock(&at->at_lock);
392 at_reset_nolock(at, timeout);
393 spin_unlock(&at->at_lock);
396 static inline void at_init(struct adaptive_timeout *at, timeout_t timeout,
399 memset(at, 0, sizeof(*at));
400 spin_lock_init(&at->at_lock);
401 at->at_flags = flags;
402 at_reset(at, timeout);
405 static inline void at_reinit(struct adaptive_timeout *at, timeout_t timeout,
408 spin_lock(&at->at_lock);
410 memset(at->at_hist, 0, sizeof(at->at_hist));
411 at->at_flags = flags;
412 at_reset_nolock(at, timeout);
413 spin_unlock(&at->at_lock);
416 timeout_t obd_at_measure(struct obd_device *obd, struct adaptive_timeout *at,
419 int import_at_get_index(struct obd_import *imp, int portal);
423 extern struct obd_import *class_exp2cliimp(struct obd_export *);
427 #endif /* __IMPORT_H */