X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Finclude%2Fobd_support.h;h=3fe05d4f1be7651087da020237b08ac2a83ce19f;hp=50379c5f558dfd400cba946e8c894a0bb723b760;hb=df19e89b6c2c5b355ab2156da99bebf939ee01fd;hpb=cefa8cda2ba2d288ccaa4ec077a6c627592503ea diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 50379c5..3fe05d4 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -1,6 +1,4 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * +/* * GPL HEADER START * * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -26,8 +24,10 @@ * GPL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved + * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. + * + * Copyright (c) 2011, 2012, Whamcloud, Inc. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -59,14 +59,6 @@ enum { OBD_STATS_NUM, }; -enum { - OBD_FAIL_LOC_NOSET = 0, - OBD_FAIL_LOC_ORSET = 1, - OBD_FAIL_LOC_RESET = 2 -}; - -extern unsigned long obd_fail_loc; -extern unsigned int obd_fail_val; extern unsigned int obd_debug_peer_on_timeout; extern unsigned int obd_dump_on_timeout; extern unsigned int obd_dump_on_eviction; @@ -74,30 +66,63 @@ extern unsigned int obd_dump_on_eviction; networking / disk / timings affected by load (use Adaptive Timeouts) */ extern unsigned int obd_timeout; /* seconds */ extern unsigned int ldlm_timeout; /* seconds */ +extern unsigned int at_min; +extern unsigned int at_max; +extern unsigned int at_history; +extern int at_early_margin; +extern int at_extra; extern unsigned int obd_sync_filter; extern unsigned int obd_max_dirty_pages; -extern atomic_t obd_dirty_pages; -extern atomic_t obd_dirty_transit_pages; -extern cfs_waitq_t obd_race_waitq; -extern int obd_race_state; +extern cfs_atomic_t obd_dirty_pages; +extern cfs_atomic_t obd_dirty_transit_pages; extern unsigned int obd_alloc_fail_rate; - -int __obd_fail_check_set(__u32 id, __u32 value, int set); -int __obd_fail_timeout_set(__u32 id, __u32 value, int ms, int set); +extern char obd_jobid_var[]; /* lvfs.c */ int obd_alloc_fail(const void *ptr, const char *name, const char *type, size_t size, const char *file, int line); +/* Some hash init argument constants */ +#define HASH_POOLS_BKT_BITS 3 +#define HASH_POOLS_CUR_BITS 3 +#define HASH_POOLS_MAX_BITS 7 +#define HASH_UUID_BKT_BITS 5 +#define HASH_UUID_CUR_BITS 7 +#define HASH_UUID_MAX_BITS 12 +#define HASH_NID_BKT_BITS 5 +#define HASH_NID_CUR_BITS 7 +#define HASH_NID_MAX_BITS 12 +#define HASH_NID_STATS_BKT_BITS 5 +#define HASH_NID_STATS_CUR_BITS 7 +#define HASH_NID_STATS_MAX_BITS 12 +#define HASH_LQS_BKT_BITS 5 +#define HASH_LQS_CUR_BITS 7 +#define HASH_LQS_MAX_BITS 12 +#define HASH_CONN_BKT_BITS 5 +#define HASH_CONN_CUR_BITS 5 +#define HASH_CONN_MAX_BITS 15 +#define HASH_EXP_LOCK_BKT_BITS 5 +#define HASH_EXP_LOCK_CUR_BITS 7 +#define HASH_EXP_LOCK_MAX_BITS 16 +#define HASH_CL_ENV_BKT_BITS 5 +#define HASH_CL_ENV_BITS 10 +#define HASH_JOB_STATS_BKT_BITS 5 +#define HASH_JOB_STATS_CUR_BITS 7 +#define HASH_JOB_STATS_MAX_BITS 12 + /* Timeout definitions */ #define OBD_TIMEOUT_DEFAULT 100 #define LDLM_TIMEOUT_DEFAULT 20 #define MDS_LDLM_TIMEOUT_DEFAULT 6 -/* Time to wait for all clients to reconnect during recovery */ +/* Time to wait for all clients to reconnect during recovery (hard limit) */ +#define OBD_RECOVERY_TIME_HARD (obd_timeout * 9) +/* Time to wait for all clients to reconnect during recovery (soft limit) */ /* Should be very conservative; must catch the first reconnect after reboot */ -#define OBD_RECOVERY_FACTOR (3) /* times obd_timeout */ +#define OBD_RECOVERY_TIME_SOFT (obd_timeout * 3) /* Change recovery-small 26b time if you change this */ #define PING_INTERVAL max(obd_timeout / 4, 1U) +/* a bit more than maximal journal commit time in seconds */ +#define PING_INTERVAL_SHORT min(PING_INTERVAL, 7U) /* Client may skip 1 ping; we must wait at least 2.5. But for multiple * failover targets the client only pings one server at a time, and pings * can be lost on a loaded network. Since eviction has serious consequences, @@ -111,20 +136,27 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type, miss the recovery window */ #define CONNECTION_SWITCH_MAX min(50U, max(CONNECTION_SWITCH_MIN,obd_timeout)) #define CONNECTION_SWITCH_INC 5 /* Connection timeout backoff */ -#ifndef CRAY_XT3 /* In general this should be low to have quick detection of a system running on a backup server. (If it's too low, import_select_connection will increase the timeout anyhow.) */ #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/20) -#else -/* ...but for very large systems (e.g. CRAY) we need to keep the initial - connect t.o. high (bz 10803), because they will nearly ALWAYS be doing the - connects for the first time (clients "reboot" after every process, so no - chance to generate adaptive timeout data. */ -#define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/2) -#endif +/* The max delay between connects is SWITCH_MAX + SWITCH_INC + INITIAL */ +#define RECONNECT_DELAY_MAX (CONNECTION_SWITCH_MAX + CONNECTION_SWITCH_INC + \ + INITIAL_CONNECT_TIMEOUT) +/* The min time a target should wait for clients to reconnect in recovery */ +#define OBD_RECOVERY_TIME_MIN (2*RECONNECT_DELAY_MAX) +#define OBD_IR_FACTOR_MIN 1 +#define OBD_IR_FACTOR_MAX 10 +#define OBD_IR_FACTOR_DEFAULT (OBD_IR_FACTOR_MAX/2) +/* default timeout for the MGS to become IR_FULL */ +#define OBD_IR_MGS_TIMEOUT (4*obd_timeout) #define LONG_UNLINK 300 /* Unlink should happen before now */ +/** + * Time interval of shrink, if the client is "idle" more than this interval, + * then the ll_grant thread will return the requested grant space to filter + */ +#define GRANT_SHRINK_INTERVAL 1200/*20 minutes*/ #define OBD_FAIL_MDS 0x100 #define OBD_FAIL_MDS_HANDLE_UNPACK 0x101 @@ -181,19 +213,33 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type, #define OBD_FAIL_MDS_SETXATTR_WRITE 0x134 #define OBD_FAIL_MDS_FS_SETUP 0x135 #define OBD_FAIL_MDS_RESEND 0x136 -#define OBD_FAIL_MDS_IS_SUBDIR_NET 0x137 -#define OBD_FAIL_MDS_IS_SUBDIR_PACK 0x138 -#define OBD_FAIL_MDS_SET_INFO_NET 0x139 -#define OBD_FAIL_MDS_WRITEPAGE_NET 0x13a -#define OBD_FAIL_MDS_WRITEPAGE_PACK 0x13b -#define OBD_FAIL_MDS_LLOG_CREATE_FAILED 0x13c -#define OBD_FAIL_MDS_OSC_PRECREATE 0x13d -#define OBD_FAIL_MDS_LOV_SYNC_RACE 0x13e -#define OBD_FAIL_MDS_CLOSE_NET_REP 0x13f -#define OBD_FAIL_MDS_LLOG_SYNC_TIMEOUT 0x140 -#define OBD_FAIL_MDS_RECOVERY_ACCEPTS_GAPS 0x141 -#define OBD_FAIL_MDS_BLOCK_QUOTA_REQ 0x142 -#define OBD_FAIL_MDS_DROP_QUOTA_REQ 0x143 +#define OBD_FAIL_MDS_LLOG_CREATE_FAILED 0x137 +#define OBD_FAIL_MDS_LOV_SYNC_RACE 0x138 +#define OBD_FAIL_MDS_OSC_PRECREATE 0x139 +#define OBD_FAIL_MDS_LLOG_SYNC_TIMEOUT 0x13a +#define OBD_FAIL_MDS_CLOSE_NET_REP 0x13b +#define OBD_FAIL_MDS_BLOCK_QUOTA_REQ 0x13c +#define OBD_FAIL_MDS_DROP_QUOTA_REQ 0x13d +#define OBD_FAIL_MDS_REMOVE_COMMON_EA 0x13e +#define OBD_FAIL_MDS_ALLOW_COMMON_EA_SETTING 0x13f +#define OBD_FAIL_MDS_FAIL_LOV_LOG_ADD 0x140 +#define OBD_FAIL_MDS_LOV_PREP_CREATE 0x141 +#define OBD_FAIL_MDS_REINT_DELAY 0x142 +#define OBD_FAIL_MDS_READLINK_EPROTO 0x143 +#define OBD_FAIL_MDS_OPEN_WAIT_CREATE 0x144 +#define OBD_FAIL_MDS_PDO_LOCK 0x145 +#define OBD_FAIL_MDS_PDO_LOCK2 0x146 +#define OBD_FAIL_MDS_OSC_CREATE_FAIL 0x147 + +/* CMD */ +#define OBD_FAIL_MDS_IS_SUBDIR_NET 0x180 +#define OBD_FAIL_MDS_IS_SUBDIR_PACK 0x181 +#define OBD_FAIL_MDS_SET_INFO_NET 0x182 +#define OBD_FAIL_MDS_WRITEPAGE_NET 0x183 +#define OBD_FAIL_MDS_WRITEPAGE_PACK 0x184 +#define OBD_FAIL_MDS_RECOVERY_ACCEPTS_GAPS 0x185 +#define OBD_FAIL_MDS_GET_INFO_NET 0x186 +#define OBD_FAIL_MDS_DQACQ_NET 0x187 #define OBD_FAIL_OST 0x200 #define OBD_FAIL_OST_CONNECT_NET 0x201 @@ -234,6 +280,10 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type, #define OBD_FAIL_OST_BRW_PAUSE_PACK 0x224 #define OBD_FAIL_OST_CONNECT_NET2 0x225 #define OBD_FAIL_OST_NOMEM 0x226 +#define OBD_FAIL_OST_BRW_PAUSE_BULK2 0x227 +#define OBD_FAIL_OST_MAPBLK_ENOSPC 0x228 +#define OBD_FAIL_OST_ENOINO 0x229 +#define OBD_FAIL_OST_DQACQ_NET 0x230 #define OBD_FAIL_LDLM 0x300 #define OBD_FAIL_LDLM_NAMESPACE_NEW 0x301 @@ -258,9 +308,14 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type, #define OBD_FAIL_LDLM_CANCEL_BL_CB_RACE 0x314 #define OBD_FAIL_LDLM_CP_CB_WAIT 0x315 #define OBD_FAIL_LDLM_OST_FAIL_RACE 0x316 +#define OBD_FAIL_LDLM_INTR_CP_AST 0x317 +#define OBD_FAIL_LDLM_CP_BL_RACE 0x318 +#define OBD_FAIL_LDLM_NEW_LOCK 0x319 +#define OBD_FAIL_LDLM_AGL_DELAY 0x31a +#define OBD_FAIL_LDLM_AGL_NOLOCK 0x31b /* LOCKLESS IO */ -#define OBD_FAIL_LDLM_SET_CONTENTION 0x315 +#define OBD_FAIL_LDLM_SET_CONTENTION 0x385 #define OBD_FAIL_OSC 0x400 #define OBD_FAIL_OSC_BRW_READ_BULK 0x401 @@ -277,6 +332,9 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type, #define OBD_FAIL_OSC_CKSUM_ADLER_ONLY 0x40c #define OBD_FAIL_OSC_DIO_PAUSE 0x40d #define OBD_FAIL_OSC_OBJECT_CONTENTION 0x40e +#define OBD_FAIL_OSC_CP_CANCEL_RACE 0x40f +#define OBD_FAIL_OSC_CP_ENQ_RACE 0x410 +#define OBD_FAIL_OSC_NO_GRANT 0x411 #define OBD_FAIL_PTLRPC 0x500 #define OBD_FAIL_PTLRPC_ACK 0x501 @@ -289,9 +347,15 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type, #define OBD_FAIL_PTLRPC_CLIENT_BULK_CB 0x508 #define OBD_FAIL_PTLRPC_PAUSE_REQ 0x50a #define OBD_FAIL_PTLRPC_PAUSE_REP 0x50c - +#define OBD_FAIL_PTLRPC_IMP_DEACTIVE 0x50d #define OBD_FAIL_PTLRPC_DUMP_LOG 0x50e -#define OBD_FAIL_PTLRPC_LONG_UNLINK 0x50f +#define OBD_FAIL_PTLRPC_LONG_REPL_UNLINK 0x50f +#define OBD_FAIL_PTLRPC_LONG_BULK_UNLINK 0x510 +#define OBD_FAIL_PTLRPC_HPREQ_TIMEOUT 0x511 +#define OBD_FAIL_PTLRPC_HPREQ_NOTIMEOUT 0x512 +#define OBD_FAIL_PTLRPC_DROP_REQ_OPC 0x513 +#define OBD_FAIL_PTLRPC_FINISH_REPLAY 0x514 +#define OBD_FAIL_PTLRPC_CLIENT_BULK_CB2 0x515 #define OBD_FAIL_OBD_PING_NET 0x600 #define OBD_FAIL_OBD_LOG_CANCEL_NET 0x601 @@ -309,6 +373,11 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type, #define OBD_FAIL_TGT_DELAY_PRECREATE 0x705 #define OBD_FAIL_TGT_TOOMANY_THREADS 0x706 #define OBD_FAIL_TGT_REPLAY_DROP 0x707 +#define OBD_FAIL_TGT_FAKE_EXP 0x708 +#define OBD_FAIL_TGT_REPLAY_DELAY 0x709 +#define OBD_FAIL_TGT_LAST_REPLAY 0x710 +#define OBD_FAIL_TGT_CLIENT_ADD 0x711 +#define OBD_FAIL_TGT_RCVG_FLAG 0x712 #define OBD_FAIL_MDC_REVALIDATE_PAUSE 0x800 #define OBD_FAIL_MDC_ENQUEUE_PAUSE 0x801 @@ -323,6 +392,8 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type, #define OBD_FAIL_MGS_PAUSE_TARGET_REG 0x905 #define OBD_FAIL_QUOTA_RET_QDATA 0xA02 +#define OBD_FAIL_QUOTA_DELAY_REL 0xA03 +#define OBD_FAIL_QUOTA_DELAY_SD 0xA04 #define OBD_FAIL_LPROC_REMOVE 0xB00 @@ -340,120 +411,38 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type, #define OBD_FAIL_SEC_CTX_FINI_NET 0x1203 #define OBD_FAIL_SEC_CTX_HDL_PAUSE 0x1204 -/* Failure injection control */ -#define OBD_FAIL_MASK_SYS 0x0000FF00 -#define OBD_FAIL_MASK_LOC (0x000000FF | OBD_FAIL_MASK_SYS) - -#define OBD_FAILED_BIT 30 -/* OBD_FAILED is 0x40000000 */ -#define OBD_FAILED (1 << OBD_FAILED_BIT) - -#define OBD_FAIL_ONCE_BIT 31 -/* OBD_FAIL_ONCE is 0x80000000 */ -#define OBD_FAIL_ONCE (1 << OBD_FAIL_ONCE_BIT) - -/* The following flags aren't made to be combined */ -#define OBD_FAIL_SKIP 0x20000000 /* skip N times then fail */ -#define OBD_FAIL_SOME 0x10000000 /* only fail N times */ -#define OBD_FAIL_RAND 0x08000000 /* fail 1/N of the times */ -#define OBD_FAIL_USR1 0x04000000 /* user flag */ - -#define OBD_FAIL_PRECHECK(id) (obd_fail_loc && \ - (obd_fail_loc & OBD_FAIL_MASK_LOC) == \ - ((id) & OBD_FAIL_MASK_LOC)) - -static inline int obd_fail_check_set(__u32 id, __u32 value, int set) -{ - int ret = 0; - if (unlikely(OBD_FAIL_PRECHECK(id) && - (ret = __obd_fail_check_set(id, value, set)))) { - CERROR("*** obd_fail_loc=%x ***\n", id); - } - return ret; -} - -/* If id hit obd_fail_loc, return 1, otherwise return 0 */ -#define OBD_FAIL_CHECK(id) \ - obd_fail_check_set(id, 0, OBD_FAIL_LOC_NOSET) - -/* If id hit obd_fail_loc, obd_fail_loc |= value and return 1, - * otherwise return 0 */ -#define OBD_FAIL_CHECK_ORSET(id, value) \ - obd_fail_check_set(id, value, OBD_FAIL_LOC_ORSET) - -/* If id hit obd_fail_loc, obd_fail_loc = value and return 1, - * otherwise return 0 */ -#define OBD_FAIL_CHECK_RESET(id, value) \ - obd_fail_check_set(id, value, OBD_FAIL_LOC_RESET) - - -static inline int obd_fail_timeout_set(__u32 id, __u32 value, int ms, int set) -{ - if (unlikely(OBD_FAIL_PRECHECK(id))) - return __obd_fail_timeout_set(id, value, ms, set); - else - return 0; -} - -/* If id hit obd_fail_loc, sleep for seconds or milliseconds */ -#define OBD_FAIL_TIMEOUT(id, secs) \ - obd_fail_timeout_set(id, 0, secs * 1000, OBD_FAIL_LOC_NOSET) - -#define OBD_FAIL_TIMEOUT_MS(id, ms) \ - obd_fail_timeout_set(id, 0, ms, OBD_FAIL_LOC_NOSET) - -/* If id hit obd_fail_loc, obd_fail_loc |= value and - * sleep seconds or milliseconds */ -#define OBD_FAIL_TIMEOUT_ORSET(id, value, secs) \ - obd_fail_timeout_set(id, value, secs * 1000, OBD_FAIL_LOC_ORSET) - -#define OBD_FAIL_TIMEOUT_MS_ORSET(id, value, ms) \ - obd_fail_timeout_set(id, value, ms, OBD_FAIL_LOC_ORSET) - -#ifdef __KERNEL__ -static inline void obd_fail_write(int id, struct super_block *sb) -{ - /* We set FAIL_ONCE because we never "un-fail" a device */ - if (OBD_FAIL_CHECK_ORSET(id & ~OBD_FAIL_ONCE, OBD_FAIL_ONCE)) { -#ifdef LIBCFS_DEBUG - BDEVNAME_DECLARE_STORAGE(tmp); - CERROR("obd_fail_loc=%x, fail write operation on %s\n", - id, ll_bdevname(sb, tmp)); -#endif - /* TODO-CMD: fix getting jdev */ - __lvfs_set_rdonly(lvfs_sbdev(sb), (lvfs_sbdev_type)0); - } -} -#define OBD_FAIL_WRITE(id, sb) obd_fail_write(id, sb) - -/* The idea here is to synchronise two threads to force a race. The - * first thread that calls this with a matching fail_loc is put to - * sleep. The next thread that calls with the same fail_loc wakes up - * the first and continues. */ -static inline void obd_race(__u32 id) -{ - if (OBD_FAIL_PRECHECK(id)) { - if (unlikely(__obd_fail_check_set(id, 0, OBD_FAIL_LOC_NOSET))) { - obd_race_state = 0; - CERROR("obd_race id %x sleeping\n", id); - OBD_SLEEP_ON(obd_race_waitq, obd_race_state != 0); - CERROR("obd_fail_race id %x awake\n", id); - } else { - CERROR("obd_fail_race id %x waking\n", id); - obd_race_state = 1; - wake_up(&obd_race_waitq); - } - } -} -#define OBD_RACE(id) obd_race(id) -#else -/* sigh. an expedient fix until OBD_RACE is fixed up */ -#define OBD_RACE(foo) do {} while(0) -#endif - -#define fixme() CDEBUG(D_OTHER, "FIXME\n"); - -extern atomic_t libcfs_kmemory; +#define OBD_FAIL_LLOG 0x1300 +#define OBD_FAIL_LLOG_ORIGIN_CONNECT_NET 0x1301 +#define OBD_FAIL_LLOG_ORIGIN_HANDLE_CREATE_NET 0x1302 +#define OBD_FAIL_LLOG_ORIGIN_HANDLE_DESTROY_NET 0x1303 +#define OBD_FAIL_LLOG_ORIGIN_HANDLE_READ_HEADER_NET 0x1304 +#define OBD_FAIL_LLOG_ORIGIN_HANDLE_NEXT_BLOCK_NET 0x1305 +#define OBD_FAIL_LLOG_ORIGIN_HANDLE_PREV_BLOCK_NET 0x1306 +#define OBD_FAIL_LLOG_ORIGIN_HANDLE_WRITE_REC_NET 0x1307 +#define OBD_FAIL_LLOG_ORIGIN_HANDLE_CLOSE_NET 0x1308 +#define OBD_FAIL_LLOG_CATINFO_NET 0x1309 +#define OBD_FAIL_MDS_SYNC_CAPA_SL 0x1310 +#define OBD_FAIL_SEQ_ALLOC 0x1311 + +#define OBD_FAIL_LLITE 0x1400 +#define OBD_FAIL_LLITE_FAULT_TRUNC_RACE 0x1401 +#define OBD_FAIL_LOCK_STATE_WAIT_INTR 0x1402 + +/* Assign references to moved code to reduce code changes */ +#define OBD_FAIL_PRECHECK(id) CFS_FAIL_PRECHECK(id) +#define OBD_FAIL_CHECK(id) CFS_FAIL_CHECK(id) +#define OBD_FAIL_CHECK_VALUE(id, value) CFS_FAIL_CHECK_VALUE(id, value) +#define OBD_FAIL_CHECK_ORSET(id, value) CFS_FAIL_CHECK_ORSET(id, value) +#define OBD_FAIL_CHECK_RESET(id, value) CFS_FAIL_CHECK_RESET(id, value) +#define OBD_FAIL_RETURN(id, ret) CFS_FAIL_RETURN(id, ret) +#define OBD_FAIL_TIMEOUT(id, secs) CFS_FAIL_TIMEOUT(id, secs) +#define OBD_FAIL_TIMEOUT_MS(id, ms) CFS_FAIL_TIMEOUT_MS(id, ms) +#define OBD_FAIL_TIMEOUT_ORSET(id, value, secs) CFS_FAIL_TIMEOUT_ORSET(id, value, secs) +#define OBD_RACE(id) CFS_RACE(id) +#define OBD_FAIL_ONCE CFS_FAIL_ONCE +#define OBD_FAILED CFS_FAILED + +extern cfs_atomic_t libcfs_kmemory; #ifdef LPROCFS #define obd_memory_add(size) \ @@ -550,18 +539,21 @@ static inline void obd_pages_sub(int order) #define OBD_ALLOC_FAIL_MULT (OBD_ALLOC_FAIL_MASK / 100) #if defined(LUSTRE_UTILS) /* this version is for utils only */ -#define OBD_ALLOC_GFP(ptr, size, gfp_mask) \ -do { \ - (ptr) = cfs_alloc(size, (gfp_mask)); \ - if (unlikely((ptr) == NULL)) { \ - CERROR("kmalloc of '" #ptr "' (%d bytes) failed at %s:%d\n", \ - (int)(size), __FILE__, __LINE__); \ - } else { \ - memset(ptr, 0, size); \ - CDEBUG(D_MALLOC, "kmalloced '" #ptr "': %d at %p\n", \ - (int)(size), ptr); \ - } \ +#define __OBD_MALLOC_VERBOSE(ptr, cptab, cpt, size, flags) \ +do { \ + (ptr) = (cptab) == NULL ? \ + cfs_alloc(size, flags) : \ + cfs_cpt_malloc(cptab, cpt, size, flags); \ + if (unlikely((ptr) == NULL)) { \ + CERROR("kmalloc of '" #ptr "' (%d bytes) failed at %s:%d\n", \ + (int)(size), __FILE__, __LINE__); \ + } else { \ + memset(ptr, 0, size); \ + CDEBUG(D_MALLOC, "kmalloced '" #ptr "': %d at %p\n", \ + (int)(size), ptr); \ + } \ } while (0) + #else /* this version is for the kernel and liblustre */ #define OBD_FREE_RTN0(ptr) \ ({ \ @@ -569,9 +561,12 @@ do { \ (ptr) = NULL; \ 0; \ }) -#define OBD_ALLOC_GFP(ptr, size, gfp_mask) \ -do { \ - (ptr) = cfs_alloc(size, (gfp_mask)); \ + +#define __OBD_MALLOC_VERBOSE(ptr, cptab, cpt, size, flags) \ +do { \ + (ptr) = (cptab) == NULL ? \ + cfs_alloc(size, flags) : \ + cfs_cpt_malloc(cptab, cpt, size, flags); \ if (likely((ptr) != NULL && \ (!HAS_FAIL_ALLOC_FLAG || obd_alloc_fail_rate == 0 || \ !obd_alloc_fail(ptr, #ptr, "km", size, \ @@ -583,33 +578,101 @@ do { \ } while (0) #endif -#ifndef OBD_ALLOC_MASK -# define OBD_ALLOC_MASK CFS_ALLOC_IO -#endif +#define OBD_ALLOC_GFP(ptr, size, gfp_mask) \ + __OBD_MALLOC_VERBOSE(ptr, NULL, 0, size, gfp_mask) -#define OBD_ALLOC(ptr, size) OBD_ALLOC_GFP(ptr, size, OBD_ALLOC_MASK) +#define OBD_ALLOC(ptr, size) OBD_ALLOC_GFP(ptr, size, CFS_ALLOC_IO) #define OBD_ALLOC_WAIT(ptr, size) OBD_ALLOC_GFP(ptr, size, CFS_ALLOC_STD) #define OBD_ALLOC_PTR(ptr) OBD_ALLOC(ptr, sizeof *(ptr)) #define OBD_ALLOC_PTR_WAIT(ptr) OBD_ALLOC_WAIT(ptr, sizeof *(ptr)) +#define OBD_CPT_ALLOC_GFP(ptr, cptab, cpt, size, gfp_mask) \ + __OBD_MALLOC_VERBOSE(ptr, cptab, cpt, size, gfp_mask) + +#define OBD_CPT_ALLOC(ptr, cptab, cpt, size) \ + OBD_CPT_ALLOC_GFP(ptr, cptab, cpt, size, CFS_ALLOC_IO) + +#define OBD_CPT_ALLOC_PTR(ptr, cptab, cpt) \ + OBD_CPT_ALLOC(ptr, cptab, cpt, sizeof *(ptr)) + #ifdef __arch_um__ -# define OBD_VMALLOC(ptr, size) OBD_ALLOC(ptr, size) -#else -# define OBD_VMALLOC(ptr, size) \ -do { \ - (ptr) = cfs_alloc_large(size); \ + +# define OBD_VMALLOC(ptr, size) \ + OBD_ALLOC(ptr, size) +# define OBD_CPT_VMALLOC(ptr, cptab, cpt, size) \ + OBD_CPT_ALLOC(ptr, cptab, cpt, size) + +#else /* !__arch_um__ */ + +# define __OBD_VMALLOC_VEROBSE(ptr, cptab, cpt, size) \ +do { \ + (ptr) = cptab == NULL ? \ + cfs_alloc_large(size) : \ + cfs_cpt_vmalloc(cptab, cpt, size); \ if (unlikely((ptr) == NULL)) { \ CERROR("vmalloc of '" #ptr "' (%d bytes) failed\n", \ (int)(size)); \ CERROR(LPU64" total bytes allocated by Lustre, %d by LNET\n", \ - obd_memory_sum(), atomic_read(&libcfs_kmemory)); \ + obd_memory_sum(), cfs_atomic_read(&libcfs_kmemory)); \ } else { \ - memset(ptr, 0, size); \ + memset(ptr, 0, size); \ OBD_ALLOC_POST(ptr, size, "vmalloced"); \ } \ } while(0) + +# define OBD_VMALLOC(ptr, size) \ + __OBD_VMALLOC_VEROBSE(ptr, NULL, 0, size) +# define OBD_CPT_VMALLOC(ptr, cptab, cpt, size) \ + __OBD_VMALLOC_VEROBSE(ptr, cptab, cpt, size) + #endif +#ifdef __KERNEL__ + +/* Allocations above this size are considered too big and could not be done + * atomically. + * + * Be very careful when changing this value, especially when decreasing it, + * since vmalloc in Linux doesn't perform well on multi-cores system, calling + * vmalloc in critical path would hurt peformance badly. See LU-66. + */ +#define OBD_ALLOC_BIG (4 * CFS_PAGE_SIZE) + +#define OBD_ALLOC_LARGE(ptr, size) \ +do { \ + if (size > OBD_ALLOC_BIG) \ + OBD_VMALLOC(ptr, size); \ + else \ + OBD_ALLOC(ptr, size); \ +} while (0) + +#define OBD_CPT_ALLOC_LARGE(ptr, cptab, cpt, size) \ +do { \ + if (size > OBD_ALLOC_BIG) \ + OBD_CPT_VMALLOC(ptr, cptab, cpt, size); \ + else \ + OBD_CPT_ALLOC(ptr, cptab, cpt, size); \ +} while (0) + +#define OBD_FREE_LARGE(ptr, size) \ +do { \ + if (size > OBD_ALLOC_BIG) \ + OBD_VFREE(ptr, size); \ + else \ + OBD_FREE(ptr, size); \ +} while (0) + +#else /* !__KERNEL__ */ + +#define OBD_ALLOC_LARGE(ptr, size) \ + OBD_ALLOC(ptr, size) +#define OBD_CPT_ALLOC_LARGE(ptr, cptab, cpt, size) \ + OBD_ALLOC(ptr, size) +#define OBD_FREE_LARGE(ptr, size) \ + OBD_FREE(ptr, size) + +#endif /* __KERNEL__ */ + #ifdef CONFIG_DEBUG_SLAB #define POISON(ptr, c, s) do {} while (0) #define POISON_PTR(ptr) ((void)0) @@ -644,22 +707,20 @@ do { \ # define my_call_rcu(rcu, cb) (cb)(rcu) #endif -#define OBD_FREE_RCU_CB(ptr, size, handle, free_cb) \ -do { \ - struct portals_handle *__h = (handle); \ - LASSERT(handle); \ - __h->h_ptr = (ptr); \ - __h->h_size = (size); \ - __h->h_free_cb = (void (*)(void *, size_t))(free_cb); \ - my_call_rcu(&__h->h_rcu, class_handle_free_cb); \ - POISON_PTR(ptr); \ +#define OBD_FREE_RCU(ptr, size, handle) \ +do { \ + struct portals_handle *__h = (handle); \ + \ + LASSERT(handle != NULL); \ + __h->h_cookie = (unsigned long)(ptr); \ + __h->h_size = (size); \ + my_call_rcu(&__h->h_rcu, class_handle_free_cb); \ + POISON_PTR(ptr); \ } while(0) -#define OBD_FREE_RCU(ptr, size, handle) OBD_FREE_RCU_CB(ptr, size, handle, NULL) #else #define OBD_FREE(ptr, size) ((void)(size), free((ptr))) #define OBD_FREE_RCU(ptr, size, handle) (OBD_FREE(ptr, size)) -#define OBD_FREE_RCU_CB(ptr, size, handle, cb) ((*(cb))(ptr, size)) #endif /* ifdef __KERNEL__ */ #ifdef __arch_um__ @@ -682,10 +743,13 @@ do { \ (ptr) = NULL; \ 0; \ }) -#define OBD_SLAB_ALLOC(ptr, slab, type, size) \ -do { \ - LASSERT(!in_interrupt()); \ - (ptr) = cfs_mem_cache_alloc(slab, (type)); \ + +#define __OBD_SLAB_ALLOC_VERBOSE(ptr, slab, cptab, cpt, size, type) \ +do { \ + LASSERT(ergo(type != CFS_ALLOC_ATOMIC, !cfs_in_interrupt())); \ + (ptr) = (cptab) == NULL ? \ + cfs_mem_cache_alloc(slab, type) : \ + cfs_mem_cache_cpt_alloc(slab, cptab, cpt, type); \ if (likely((ptr) != NULL && \ (!HAS_FAIL_ALLOC_FLAG || obd_alloc_fail_rate == 0 || \ !obd_alloc_fail(ptr, #ptr, "slab-", size, \ @@ -696,6 +760,11 @@ do { \ } \ } while(0) +#define OBD_SLAB_ALLOC_GFP(ptr, slab, size, flags) \ + __OBD_SLAB_ALLOC_VERBOSE(ptr, slab, NULL, 0, size, flags) +#define OBD_SLAB_CPT_ALLOC_GFP(ptr, slab, cptab, cpt, size, flags) \ + __OBD_SLAB_ALLOC_VERBOSE(ptr, slab, cptab, cpt, size, flags) + #define OBD_FREE_PTR(ptr) OBD_FREE(ptr, sizeof *(ptr)) #define OBD_SLAB_FREE(ptr, slab, size) \ @@ -705,53 +774,71 @@ do { \ POISON_PTR(ptr); \ } while(0) -#define OBD_SLAB_ALLOC_PTR(ptr, slab) \ - OBD_SLAB_ALLOC((ptr), (slab), CFS_ALLOC_STD, sizeof *(ptr)) -#define OBD_SLAB_FREE_PTR(ptr, slab) \ - OBD_SLAB_FREE((ptr), (slab), sizeof *(ptr)) +#define OBD_SLAB_ALLOC(ptr, slab, size) \ + OBD_SLAB_ALLOC_GFP(ptr, slab, size, CFS_ALLOC_IO) + +#define OBD_SLAB_CPT_ALLOC(ptr, slab, cptab, cpt, size) \ + OBD_SLAB_CPT_ALLOC_GFP(ptr, slab, cptab, cpt, size, CFS_ALLOC_IO) + +#define OBD_SLAB_ALLOC_PTR(ptr, slab) \ + OBD_SLAB_ALLOC(ptr, slab, sizeof *(ptr)) + +#define OBD_SLAB_CPT_ALLOC_PTR(ptr, slab, cptab, cpt) \ + OBD_SLAB_CPT_ALLOC(ptr, slab, cptab, cpt, sizeof *(ptr)) + +#define OBD_SLAB_ALLOC_PTR_GFP(ptr, slab, flags) \ + OBD_SLAB_ALLOC_GFP(ptr, slab, sizeof *(ptr), flags) + +#define OBD_SLAB_CPT_ALLOC_PTR_GFP(ptr, slab, ctab, cpt, flags) \ + OBD_SLAB_CPT_ALLOC_GFP(ptr, slab, cptab, cpt, sizeof *(ptr), flags) + +#define OBD_SLAB_FREE_PTR(ptr, slab) \ + OBD_SLAB_FREE((ptr), (slab), sizeof *(ptr)) #define KEY_IS(str) \ (keylen >= (sizeof(str)-1) && memcmp(key, str, (sizeof(str)-1)) == 0) /* Wrapper for contiguous page frame allocation */ -#define OBD_PAGES_ALLOC(ptr, order, gfp_mask) \ -do { \ - (ptr) = cfs_alloc_pages(gfp_mask, order); \ +#define __OBD_PAGE_ALLOC_VERBOSE(ptr, cptab, cpt, gfp_mask) \ +do { \ + (ptr) = (cptab) == NULL ? \ + cfs_alloc_page(gfp_mask) : \ + cfs_page_cpt_alloc(cptab, cpt, gfp_mask); \ if (unlikely((ptr) == NULL)) { \ CERROR("alloc_pages of '" #ptr "' %d page(s) / "LPU64" bytes "\ - "failed\n", (int)(1 << (order)), \ - (__u64)((1 << (order)) << CFS_PAGE_SHIFT)); \ + "failed\n", (int)1, \ + (__u64)(1 << CFS_PAGE_SHIFT)); \ CERROR(LPU64" total bytes and "LPU64" total pages " \ "("LPU64" bytes) allocated by Lustre, " \ "%d total bytes by LNET\n", \ obd_memory_sum(), \ obd_pages_sum() << CFS_PAGE_SHIFT, \ obd_pages_sum(), \ - atomic_read(&libcfs_kmemory)); \ + cfs_atomic_read(&libcfs_kmemory)); \ } else { \ - obd_pages_add(order); \ + obd_pages_add(0); \ CDEBUG(D_MALLOC, "alloc_pages '" #ptr "': %d page(s) / " \ LPU64" bytes at %p.\n", \ - (int)(1 << (order)), \ - (__u64)((1 << (order)) << CFS_PAGE_SHIFT), ptr); \ + (int)1, \ + (__u64)(1 << CFS_PAGE_SHIFT), ptr); \ } \ } while (0) -#define OBD_PAGE_ALLOC(ptr, gfp_mask) \ - OBD_PAGES_ALLOC(ptr, 0, gfp_mask) +#define OBD_PAGE_ALLOC(ptr, gfp_mask) \ + __OBD_PAGE_ALLOC_VERBOSE(ptr, NULL, 0, gfp_mask) +#define OBD_PAGE_CPT_ALLOC(ptr, cptab, cpt, gfp_mask) \ + __OBD_PAGE_ALLOC_VERBOSE(ptr, cptab, cpt, gfp_mask) -#define OBD_PAGES_FREE(ptr, order) \ +#define OBD_PAGE_FREE(ptr) \ do { \ LASSERT(ptr); \ - obd_pages_sub(order); \ + obd_pages_sub(0); \ CDEBUG(D_MALLOC, "free_pages '" #ptr "': %d page(s) / "LPU64" bytes " \ "at %p.\n", \ - (int)(1 << (order)), (__u64)((1 << (order)) << CFS_PAGE_SHIFT),\ + (int)1, (__u64)(1 << CFS_PAGE_SHIFT), \ ptr); \ - __cfs_free_pages(ptr, order); \ + cfs_free_page(ptr); \ (ptr) = (void *)0xdeadbeef; \ } while (0) -#define OBD_PAGE_FREE(ptr) OBD_PAGES_FREE(ptr, 0) - #endif