lustre/include/lustre_import.h

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2011, 2016, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  */
  31 /** \defgroup obd_import PtlRPC import definitions
  32  * Imports are client-side representation of remote obd target.
  33  *
  34  * @{
  35  */
  36
  37 #ifndef __IMPORT_H
  38 #define __IMPORT_H
  39
  40 /** \defgroup export export
  41  *
  42  * @{
  43  */
  44 #include <linux/atomic.h>
  45 #include <linux/list.h>
  46 #include <linux/mutex.h>
  47 #include <linux/refcount.h>
  48 #include <linux/spinlock.h>
  49 #include <linux/time.h>
  50 #include <linux/types.h>
  51 #include <linux/workqueue.h>
  52 #include <libcfs/libcfs.h>
  53 #include <uapi/linux/lustre/lustre_idl.h>
  54
  55 /**
  56  * Adaptive Timeout stuff
  57  *
  58  * @{
  59  */
  60 #define D_ADAPTTO D_OTHER
  61 #define AT_BINS 4                  /* "bin" means "N seconds of history" */
  62 #define AT_FLG_NOHIST 0x1          /* use last reported value only */
  63
  64 struct adaptive_timeout {
  65         time64_t        at_binstart;         /* bin start time */
  66         unsigned int    at_hist[AT_BINS];    /* timeout history bins */
  67         unsigned int    at_flags;
  68         timeout_t       at_current_timeout;     /* current timeout value */
  69         timeout_t       at_worst_timeout_ever;  /* worst-ever timeout delta
  70                                                  * value
  71                                                  */
  72         time64_t        at_worst_timestamp;     /* worst-ever timeout
  73                                                  * timestamp
  74                                                  */
  75         spinlock_t      at_lock;
  76 };
  77
  78 enum lustre_at_flags {
  79         LATF_SKIP       = 0x0,
  80         LATF_STATS      = 0x1,
  81 };
  82
  83 struct ptlrpc_at_array {
  84         struct list_head *paa_reqs_array; /** array to hold requests */
  85         __u32             paa_size;       /** the size of array */
  86         __u32             paa_count;      /** the total count of reqs */
  87         time64_t          paa_deadline;   /** the earliest deadline of reqs */
  88         __u32            *paa_reqs_count; /** the count of reqs in each entry */
  89 };
  90
  91 #define IMP_AT_MAX_PORTALS 8
  92 struct imp_at {
  93         int                     iat_portal[IMP_AT_MAX_PORTALS];
  94         struct adaptive_timeout iat_net_latency;
  95         struct adaptive_timeout iat_service_estimate[IMP_AT_MAX_PORTALS];
  96 };
  97
  98
  99 /** @} */
 100
 101 /** Possible import states */
 102 enum lustre_imp_state {
 103         LUSTRE_IMP_CLOSED     = 1,
 104         LUSTRE_IMP_NEW        = 2,
 105         LUSTRE_IMP_DISCON     = 3,
 106         LUSTRE_IMP_CONNECTING = 4,
 107         LUSTRE_IMP_REPLAY     = 5,
 108         LUSTRE_IMP_REPLAY_LOCKS = 6,
 109         LUSTRE_IMP_REPLAY_WAIT  = 7,
 110         LUSTRE_IMP_RECOVER    = 8,
 111         LUSTRE_IMP_FULL       = 9,
 112         LUSTRE_IMP_EVICTED    = 10,
 113         LUSTRE_IMP_IDLE       = 11,
 114         LUSTRE_IMP_LAST
 115 };
 116
 117 /** Returns test string representation of numeric import state \a state */
 118 static inline const char *ptlrpc_import_state_name(enum lustre_imp_state state)
 119 {
 120         static const char * const import_state_names[] = {
 121                 "<UNKNOWN>", "CLOSED",  "NEW", "DISCONN",
 122                 "CONNECTING", "REPLAY", "REPLAY_LOCKS", "REPLAY_WAIT",
 123                 "RECOVER", "FULL", "EVICTED", "IDLE",
 124         };
 125
 126         LASSERT(state < LUSTRE_IMP_LAST);
 127         return import_state_names[state];
 128 }
 129
 130 /**
 131  * List of import event types
 132  */
 133 enum obd_import_event {
 134         IMP_EVENT_DISCON     = 0x808001,
 135         IMP_EVENT_INACTIVE   = 0x808002,
 136         IMP_EVENT_INVALIDATE = 0x808003,
 137         IMP_EVENT_ACTIVE     = 0x808004,
 138         IMP_EVENT_OCD        = 0x808005,
 139         IMP_EVENT_DEACTIVATE = 0x808006,
 140         IMP_EVENT_ACTIVATE   = 0x808007,
 141 };
 142
 143 /**
 144  * Definition of import connection structure
 145  */
 146 struct obd_import_conn {
 147         /** Item for linking connections together */
 148         struct list_head          oic_item;
 149         /** Pointer to actual PortalRPC connection */
 150         struct ptlrpc_connection *oic_conn;
 151         /** uuid of remote side */
 152         struct obd_uuid           oic_uuid;
 153         /**
 154          * Time (64 bit seconds) of last connection attempt on this connection
 155          */
 156         time64_t                  oic_last_attempt;
 157         unsigned int              oic_attempts;
 158         unsigned int              oic_replied;
 159         bool                      oic_uptodate;
 160 };
 161
 162 /* state history */
 163 #define IMP_STATE_HIST_LEN 16
 164 struct import_state_hist {
 165         enum lustre_imp_state   ish_state;
 166         time64_t                ish_time;
 167 };
 168
 169 /**
 170  * Defintion of PortalRPC import structure.
 171  * Imports are representing client-side view to remote target.
 172  */
 173 struct obd_import {
 174         /** Reference counter */
 175         refcount_t                imp_refcount;
 176         struct lustre_handle      imp_dlm_handle; /* client's ldlm export */
 177         /** Currently active connection */
 178         struct ptlrpc_connection *imp_connection;
 179         /** PortalRPC client structure for this import */
 180         struct ptlrpc_client     *imp_client;
 181         /** List element for linking into pinger chain */
 182         struct list_head          imp_pinger_chain;
 183         /** work struct for destruction of import */
 184         struct work_struct        imp_zombie_work;
 185
 186         /**
 187          * Lists of requests that are retained for replay, waiting for a reply,
 188          * or waiting for recovery to complete, respectively.
 189          * @{
 190          */
 191         struct list_head        imp_replay_list;
 192         struct list_head        imp_sending_list;
 193         struct list_head        imp_delayed_list;
 194         /** @} */
 195
 196         /**
 197          * List of requests that are retained for committed open replay. Once
 198          * open is committed, open replay request will be moved from the
 199          * imp_replay_list into the imp_committed_list.
 200          * The imp_replay_cursor is for accelerating searching during replay.
 201          * @{
 202          */
 203         struct list_head        imp_committed_list;
 204         struct list_head        *imp_replay_cursor;
 205         /** @} */
 206
 207         /** List of not replied requests */
 208         struct list_head        imp_unreplied_list;
 209         /** XID below which we know all replies have been received */
 210         __u64                   imp_known_replied_xid;
 211         /** highest XID for which we have received a reply */
 212         __u64                   imp_highest_replied_xid;
 213
 214         /** obd device for this import */
 215         struct obd_device       *imp_obd;
 216
 217         /**
 218          * some seciruty-related fields
 219          * @{
 220          */
 221         struct ptlrpc_sec        *imp_sec;
 222         rwlock_t                  imp_sec_lock;
 223         time64_t                imp_sec_expire;
 224         pid_t                     imp_sec_refpid;
 225         /** @} */
 226
 227         /** Wait queue for those who need to wait for recovery completion */
 228         wait_queue_head_t         imp_recovery_waitq;
 229
 230         /** Number of requests allocated */
 231         atomic_t                  imp_reqs;
 232         /** Number of requests currently in-flight */
 233         atomic_t                  imp_inflight;
 234         /** Number of requests currently unregistering */
 235         atomic_t                  imp_unregistering;
 236         /** Number of replay requests inflight */
 237         atomic_t                  imp_replay_inflight;
 238         /** In-flight replays rate control */
 239         wait_queue_head_t         imp_replay_waitq;
 240
 241         /** Number of currently happening import invalidations */
 242         atomic_t                  imp_inval_count;
 243         /** Numbner of request timeouts */
 244         atomic_t                  imp_timeouts;
 245         /** Current import state */
 246         enum lustre_imp_state     imp_state;
 247         /** Last replay state */
 248         enum lustre_imp_state     imp_replay_state;
 249         /** History of import states */
 250         struct import_state_hist  imp_state_hist[IMP_STATE_HIST_LEN];
 251         int                       imp_state_hist_idx;
 252         /** Current import generation. Incremented on every reconnect */
 253         int                       imp_generation;
 254         /** Idle connection initiated at this generation */
 255         int                       imp_initiated_at;
 256         /** Incremented every time we send reconnection request */
 257         __u32                     imp_conn_cnt;
 258        /**
 259         * \see ptlrpc_free_committed remembers imp_generation value here
 260         * after a check to save on unnecessary replay list iterations
 261         */
 262         int                       imp_last_generation_checked;
 263         /** Last tranno we replayed */
 264         __u64                     imp_last_replay_transno;
 265         /** Last transno committed on remote side */
 266         __u64                     imp_peer_committed_transno;
 267         /**
 268          * \see ptlrpc_free_committed remembers last_transno since its last
 269          * check here and if last_transno did not change since last run of
 270          * ptlrpc_free_committed and import generation is the same, we can
 271          * skip looking for requests to remove from replay list as optimisation
 272          */
 273         __u64                     imp_last_transno_checked;
 274         /**
 275          * Remote export handle. This is how remote side knows what export
 276          * we are talking to. Filled from response to connect request
 277          */
 278         struct lustre_handle      imp_remote_handle;
 279         /** When to perform next ping. time in jiffies. */
 280         time64_t                imp_next_ping;
 281         /** When we last successfully connected. time in 64bit jiffies */
 282         time64_t                imp_last_success_conn;
 283
 284         /** List of all possible connection for import. */
 285         struct list_head        imp_conn_list;
 286         /**
 287          * Current connection. \a imp_connection is imp_conn_current->oic_conn
 288          */
 289         struct obd_import_conn   *imp_conn_current;
 290
 291         /** Protects flags, level, generation, conn_cnt, *_list */
 292         spinlock_t                imp_lock;
 293
 294         /**
 295          * A "sentinel" value used to check if there are other threads
 296          * waiting on the imp_lock.
 297          */
 298         atomic_t                  imp_waiting;
 299
 300         /* flags */
 301         unsigned long             imp_invalid:1,    /* evicted */
 302                                   /* administratively disabled */
 303                                   imp_deactive:1,
 304                                   /* try to recover the import */
 305                                   imp_replayable:1,
 306                                   /* don't run recovery (timeout instead) */
 307                                   imp_dlm_fake:1,
 308                                   /* use 1/2 timeout on MDS' OSCs */
 309                                   imp_server_timeout:1,
 310                                   /* VBR: imp in delayed recovery */
 311                                   imp_delayed_recovery:1,
 312                                   /* recovery by versions was failed */
 313                                   imp_vbr_failed:1,
 314                                   /* force an immidiate ping */
 315                                   imp_force_verify:1,
 316                                   /* force a scheduled ping */
 317                                   imp_force_next_verify:1,
 318                                   /* pingable */
 319                                   imp_pingable:1,
 320                                   /* resend for replay */
 321                                   imp_resend_replay:1,
 322                                   /* disable normal recovery, for test only. */
 323                                   imp_no_pinger_recover:1,
 324                                   /* import must be reconnected instead of
 325                                    * chouse new connection */
 326                                   imp_force_reconnect:1,
 327                                   /* import has tried to connect with server */
 328                                   imp_connect_tried:1,
 329                                   /* connected but not FULL yet */
 330                                   imp_connected:1,
 331                                   /* grant shrink disabled */
 332                                   imp_grant_shrink_disabled:1,
 333                                   /* to supress LCONSOLE() at conn.restore */
 334                                   imp_was_idle:1,
 335                                   imp_no_cached_data:1;
 336         u32                       imp_connect_op;
 337         u32                       imp_idle_timeout;
 338         u32                       imp_idle_debug;
 339         struct obd_connect_data   imp_connect_data;
 340         __u64                     imp_connect_flags_orig;
 341         __u64                     imp_connect_flags2_orig;
 342         int                       imp_connect_error;
 343
 344         enum lustre_msg_magic   imp_msg_magic;
 345                                 /* adjusted based on server capability */
 346         enum lustre_msghdr      imp_msghdr_flags;
 347
 348                                 /* adaptive timeout data */
 349         struct imp_at           imp_at;
 350         time64_t                imp_last_reply_time;    /* for health check */
 351         time64_t                imp_setup_time;
 352         __u32                   imp_conn_restricted_net;
 353 };
 354
 355 /* import.c : adaptive timeout handling.
 356  *
 357  * Lustre tracks how long RPCs take to complete. This information is reported
 358  * back to clients who utilize the information to estimate the time needed
 359  * for future requests and set appropriate RPC timeouts. Minimum and maximum
 360  * service times can be configured via the at_min and at_max kernel module
 361  * parameters, respectively.
 362  *
 363  * Since this information is transmitted between nodes the timeouts are in
 364  * seconds not jiffies which can vary from node to node. To avoid confusion
 365  * the timeout is handled in timeout_t (s32) instead of time64_t or
 366  * long (jiffies).
 367  */
 368 static inline timeout_t at_est2timeout(timeout_t timeout)
 369 {
 370         /* add an arbitrary minimum: 125% +5 sec */
 371         return timeout + (timeout >> 2) + 5;
 372 }
 373
 374 static inline timeout_t at_timeout2est(timeout_t timeout)
 375 {
 376         /* restore estimate value from timeout: e=4/5(t-5) */
 377         LASSERT(timeout > 0);
 378         return max((timeout << 2) / 5, 5) - 4;
 379 }
 380
 381 static inline void at_reset_nolock(struct adaptive_timeout *at,
 382                                    timeout_t timeout)
 383 {
 384         at->at_current_timeout = timeout;
 385         at->at_worst_timeout_ever = timeout;
 386         at->at_worst_timestamp = ktime_get_real_seconds();
 387 }
 388
 389 static inline void at_reset(struct adaptive_timeout *at, timeout_t timeout)
 390 {
 391         spin_lock(&at->at_lock);
 392         at_reset_nolock(at, timeout);
 393         spin_unlock(&at->at_lock);
 394 }
 395
 396 static inline void at_init(struct adaptive_timeout *at, timeout_t timeout,
 397                            int flags)
 398 {
 399         memset(at, 0, sizeof(*at));
 400         spin_lock_init(&at->at_lock);
 401         at->at_flags = flags;
 402         at_reset(at, timeout);
 403 }
 404
 405 static inline void at_reinit(struct adaptive_timeout *at, timeout_t timeout,
 406                              int flags)
 407 {
 408         spin_lock(&at->at_lock);
 409         at->at_binstart = 0;
 410         memset(at->at_hist, 0, sizeof(at->at_hist));
 411         at->at_flags = flags;
 412         at_reset_nolock(at, timeout);
 413         spin_unlock(&at->at_lock);
 414 }
 415
 416 timeout_t obd_at_measure(struct obd_device *obd, struct adaptive_timeout *at,
 417                             timeout_t timeout);
 418
 419 int import_at_get_index(struct obd_import *imp, int portal);
 420
 421 /* genops.c */
 422 struct obd_export;
 423 extern struct obd_import *class_exp2cliimp(struct obd_export *);
 424
 425 /** @} import */
 426
 427 #endif /* __IMPORT_H */
 428
 429 /** @} obd_import */