lustre/include/lustre_nrs_tbf.h

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License version 2 for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; If not, see
  17  * http://www.gnu.org/licenses/gpl-2.0.html
  18  *
  19  * GPL HEADER END
  20  */
  21 /*
  22  * Copyright (C) 2013 DataDirect Networks, Inc.
  23  *
  24  * Copyright (c) 2014, Intel Corporation.
  25  */
  26 /*
  27  *
  28  * Network Request Scheduler (NRS) Token Bucket Filter(TBF) policy
  29  *
  30  */
  31
  32 #ifndef _LUSTRE_NRS_TBF_H
  33 #define _LUSTRE_NRS_TBF_H
  34
  35 /* \name tbf
  36  *
  37  * TBF policy
  38  *
  39  * @{
  40  */
  41
  42 struct nrs_tbf_head;
  43 struct nrs_tbf_cmd;
  44
  45 #define NRS_TBF_MATCH_FULL      0x0000001
  46 #define NRS_TBF_MATCH_WILDCARD  0x0000002
  47
  48 struct nrs_tbf_jobid {
  49         char            *tj_id;
  50         __u32            tj_match_flag;
  51         struct list_head tj_linkage;
  52 };
  53
  54 #define MAX_U32_STR_LEN 10
  55 #define NRS_TBF_KEY_LEN (LNET_NIDSTR_SIZE + LUSTRE_JOBID_SIZE + \
  56                          MAX_U32_STR_LEN + MAX_U32_STR_LEN + 3 + 2)
  57
  58 enum nrs_tbf_flag {
  59         NRS_TBF_FLAG_INVALID    = 0x0000000,
  60         NRS_TBF_FLAG_JOBID      = 0x0000001,
  61         NRS_TBF_FLAG_NID        = 0x0000002,
  62         NRS_TBF_FLAG_OPCODE     = 0x0000004,
  63         NRS_TBF_FLAG_GENERIC    = 0x0000008,
  64         NRS_TBF_FLAG_UID        = 0x0000010,
  65         NRS_TBF_FLAG_GID        = 0x0000020,
  66 };
  67
  68 struct tbf_id {
  69         enum nrs_tbf_flag       ti_type;
  70         u32                     ti_uid;
  71         u32                     ti_gid;
  72 };
  73
  74 struct nrs_tbf_id {
  75         struct tbf_id           nti_id;
  76         struct list_head        nti_linkage;
  77 };
  78
  79 struct nrs_tbf_client {
  80         /** Resource object for policy instance. */
  81         struct ptlrpc_nrs_resource       tc_res;
  82         /** Node in the hash table. */
  83         struct rhash_head                tc_rhash;
  84         struct hlist_node                tc_hnode;
  85         /** NID of the client. */
  86         struct lnet_nid                  tc_nid;
  87         /** Jobid of the client. */
  88         char                             tc_jobid[LUSTRE_JOBID_SIZE];
  89         /** opcode of the client. */
  90         __u32                            tc_opcode;
  91         /** gid or uid of the client. */
  92         struct tbf_id                   tc_id;
  93         /** Hash key of the client. */
  94         char                             tc_key[NRS_TBF_KEY_LEN];
  95         /** Reference number of the client. */
  96         refcount_t                       tc_ref;
  97         /** Lock to protect rule and linkage. */
  98         spinlock_t                       tc_rule_lock;
  99         /** Linkage to rule. */
 100         struct list_head                 tc_linkage;
 101         /** Pointer to rule. */
 102         struct nrs_tbf_rule             *tc_rule;
 103         /** Generation of the rule matched. */
 104         __u64                            tc_rule_generation;
 105         /** Limit of RPC rate. */
 106         __u64                            tc_rpc_rate;
 107         /** Time to wait for next token. */
 108         __u64                            tc_nsecs;
 109         /** RPC token number. */
 110         __u64                            tc_ntoken;
 111         /** Token bucket depth. */
 112         __u64                            tc_depth;
 113         /** Time check-point. */
 114         __u64                            tc_check_time;
 115         /** Deadline of a class */
 116         __u64                            tc_deadline;
 117         /**
 118          * Time residue: the remainder of elapsed time
 119          * divided by nsecs when dequeue a request.
 120          */
 121         __u64                            tc_nsecs_resid;
 122         /** List of queued requests. */
 123         struct list_head                 tc_list;
 124         /** Node in binary heap. */
 125         struct binheap_node              tc_node;
 126         /** Whether the client is in heap. */
 127         bool                             tc_in_heap;
 128         /** Sequence of the newest rule. */
 129         __u32                            tc_rule_sequence;
 130         /**
 131          * Linkage into LRU list. Protected bucket lock of
 132          * nrs_tbf_head::th_cli_hash.
 133          */
 134         struct list_head                 tc_lru;
 135         /**
 136          * RCU head for rhashtable handling
 137          */
 138         struct rcu_head                  tc_rcu_head;
 139 };
 140
 141 #define MAX_TBF_NAME (16)
 142
 143 enum nrs_rule_flags {
 144         NTRS_STOPPING   = 0x00000001,
 145         NTRS_DEFAULT    = 0x00000002,
 146         NTRS_REALTIME   = 0x00000004,
 147 };
 148
 149 struct nrs_tbf_rule {
 150         /** Name of the rule. */
 151         char                             tr_name[MAX_TBF_NAME];
 152         /** Head belongs to. */
 153         struct nrs_tbf_head             *tr_head;
 154         /** Likage to head. */
 155         struct list_head                 tr_linkage;
 156         /** Nid list of the rule. */
 157         struct list_head                 tr_nids;
 158         /** Nid list string of the rule.*/
 159         char                            *tr_nids_str;
 160         /** Jobid list of the rule. */
 161         struct list_head                 tr_jobids;
 162         /** Jobid list string of the rule.*/
 163         char                            *tr_jobids_str;
 164         /** uid/gid list of the rule. */
 165         struct list_head                tr_ids;
 166         /** uid/gid list string of the rule. */
 167         char                            *tr_ids_str;
 168         /** Opcode bitmap of the rule. */
 169         unsigned long                   *tr_opcodes;
 170         u32                             tr_opcodes_cnt;
 171         /** Opcode list string of the rule.*/
 172         char                            *tr_opcodes_str;
 173         /** Condition list of the rule.*/
 174         struct list_head                tr_conds;
 175         /** Generic condition string of the rule. */
 176         char                            *tr_conds_str;
 177         /** RPC/s limit. */
 178         __u64                            tr_rpc_rate;
 179         /** Time to wait for next token. */
 180         u64                              tr_nsecs_per_rpc;
 181         /** Token bucket depth. */
 182         __u64                            tr_depth;
 183         /** Lock to protect the list of clients. */
 184         spinlock_t                       tr_rule_lock;
 185         /** List of client. */
 186         struct list_head                 tr_cli_list;
 187         /** Flags of the rule. */
 188         enum nrs_rule_flags              tr_flags;
 189         /** Usage Reference count taken on the rule. */
 190         atomic_t                         tr_ref;
 191         /** Generation of the rule. */
 192         __u64                            tr_generation;
 193 };
 194
 195 struct nrs_tbf_ops {
 196         char *o_name;
 197         int (*o_startup)(struct ptlrpc_nrs_policy *, struct nrs_tbf_head *);
 198         struct nrs_tbf_client *(*o_cli_find)(struct nrs_tbf_head *,
 199                                              struct ptlrpc_request *);
 200         struct nrs_tbf_client *(*o_cli_findadd)(struct nrs_tbf_head *,
 201                                                 struct nrs_tbf_client *);
 202         void (*o_cli_put)(struct nrs_tbf_head *, struct nrs_tbf_client *);
 203         void (*o_cli_init)(struct nrs_tbf_client *, struct ptlrpc_request *);
 204         int (*o_rule_init)(struct ptlrpc_nrs_policy *,
 205                            struct nrs_tbf_rule *,
 206                            struct nrs_tbf_cmd *);
 207         int (*o_rule_dump)(struct nrs_tbf_rule *, struct seq_file *);
 208         int (*o_rule_match)(struct nrs_tbf_rule *,
 209                             struct nrs_tbf_client *);
 210         void (*o_rule_fini)(struct nrs_tbf_rule *);
 211 };
 212
 213 #define NRS_TBF_TYPE_JOBID      "jobid"
 214 #define NRS_TBF_TYPE_NID        "nid"
 215 #define NRS_TBF_TYPE_OPCODE     "opcode"
 216 #define NRS_TBF_TYPE_GENERIC    "generic"
 217 #define NRS_TBF_TYPE_UID        "uid"
 218 #define NRS_TBF_TYPE_GID        "gid"
 219 #define NRS_TBF_TYPE_MAX_LEN    20
 220
 221 struct nrs_tbf_type {
 222         const char              *ntt_name;
 223         enum nrs_tbf_flag        ntt_flag;
 224         struct nrs_tbf_ops      *ntt_ops;
 225 };
 226
 227 struct nrs_tbf_bucket {
 228         /**
 229          * LRU list, updated on each access to client. Protected by
 230          * bucket lock of nrs_tbf_head::th_cli_hash.
 231          */
 232         struct list_head        ntb_lru;
 233 };
 234
 235 /**
 236  * Private data structure for the TBF policy
 237  */
 238 struct nrs_tbf_head {
 239         /**
 240          * Resource object for policy instance.
 241          */
 242         struct ptlrpc_nrs_resource       th_res;
 243         /**
 244          * Hash of clients.
 245          */
 246         struct rhashtable                th_cli_rhash ____cacheline_aligned_in_smp;
 247         /**
 248          * List of rules.
 249          */
 250         struct list_head                 th_list;
 251         /**
 252          * Lock to protect the list of rules.
 253          */
 254         spinlock_t                       th_rule_lock;
 255         /**
 256          * Generation of rules.
 257          */
 258         atomic_t                         th_rule_sequence;
 259         /**
 260          * Default rule.
 261          */
 262         struct nrs_tbf_rule             *th_rule;
 263         /**
 264          * Timer for next token.
 265          */
 266         struct hrtimer                   th_timer;
 267         /**
 268          * Deadline of the timer.
 269          */
 270         __u64                            th_deadline;
 271         /**
 272          * Sequence of requests.
 273          */
 274         __u64                            th_sequence;
 275         /**
 276          * Heap of queues.
 277          */
 278         struct binheap                  *th_binheap;
 279         /**
 280          * Hash of clients.
 281          */
 282         struct cfs_hash                 *th_cli_hash;
 283         /**
 284          * Type of TBF policy.
 285          */
 286         char                             th_type[NRS_TBF_TYPE_MAX_LEN + 1];
 287         /**
 288          * Rule operations.
 289          */
 290         struct nrs_tbf_ops              *th_ops;
 291         /**
 292          * Flag of type.
 293          */
 294         __u32                            th_type_flag;
 295         /**
 296          * Index of bucket on hash table while purging.
 297          */
 298         int                              th_purge_start;
 299 };
 300
 301 enum nrs_tbf_cmd_type {
 302         NRS_CTL_TBF_START_RULE = 0,
 303         NRS_CTL_TBF_STOP_RULE,
 304         NRS_CTL_TBF_CHANGE_RULE,
 305 };
 306
 307 struct nrs_tbf_cmd {
 308         enum nrs_tbf_cmd_type                    tc_cmd;
 309         char                                    *tc_name;
 310         union {
 311                 struct nrs_tbf_cmd_start {
 312                         __u64                    ts_rpc_rate;
 313                         struct list_head         ts_nids;
 314                         char                    *ts_nids_str;
 315                         struct list_head         ts_jobids;
 316                         char                    *ts_jobids_str;
 317                         struct list_head         ts_ids;
 318                         char                    *ts_ids_str;
 319                         char                    *ts_opcodes_str;
 320                         struct list_head         ts_conds;
 321                         char                    *ts_conds_str;
 322                         __u32                    ts_valid_type;
 323                         enum nrs_rule_flags      ts_rule_flags;
 324                         char                    *ts_next_name;
 325                 } tc_start;
 326                 struct nrs_tbf_cmd_change {
 327                         __u64                    tc_rpc_rate;
 328                         char                    *tc_next_name;
 329                 } tc_change;
 330         } u;
 331 };
 332
 333 enum nrs_tbf_field {
 334         NRS_TBF_FIELD_NID,
 335         NRS_TBF_FIELD_JOBID,
 336         NRS_TBF_FIELD_OPCODE,
 337         NRS_TBF_FIELD_UID,
 338         NRS_TBF_FIELD_GID,
 339         NRS_TBF_FIELD_MAX
 340 };
 341
 342 struct nrs_tbf_expression {
 343         enum nrs_tbf_field       te_field;
 344         struct list_head         te_cond;
 345         unsigned long           *te_opcodes;
 346         u32                      te_opcodes_cnt;
 347         struct list_head         te_linkage;
 348 };
 349
 350 struct nrs_tbf_conjunction {
 351         /**
 352          * link to disjunction.
 353          */
 354         struct list_head tc_linkage;
 355         /**
 356          * list of logical conjunction
 357          */
 358         struct list_head tc_expressions;
 359 };
 360
 361 struct nrs_tbf_req {
 362         /**
 363          * Linkage to queue.
 364          */
 365         struct list_head        tr_list;
 366         /**
 367          * Sequence of the request.
 368          */
 369         __u64                   tr_sequence;
 370 };
 371
 372 /**
 373  * TBF policy operations.
 374  *
 375  * Read the the data of a TBF policy.
 376  */
 377 #define NRS_CTL_TBF_RD_RULE PTLRPC_NRS_CTL_POL_SPEC_01
 378 /**
 379  * Write the the data of a TBF policy.
 380  */
 381 #define NRS_CTL_TBF_WR_RULE PTLRPC_NRS_CTL_POL_SPEC_02
 382 /**
 383  * Read the TBF policy type preset by proc entry "nrs_policies".
 384  */
 385 #define NRS_CTL_TBF_RD_TYPE_FLAG PTLRPC_NRS_CTL_POL_SPEC_03
 386
 387 /** @} tbf */
 388 #endif