* Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
* Use is subject to license terms.
*
- * Copyright (c) 2011, 2012, Intel Corporation.
+ * Copyright (c) 2011, 2014, Intel Corporation.
*/
/*
* This file is part of Lustre, http://www.lustre.org/
#ifndef _OBD_SUPPORT
#define _OBD_SUPPORT
+#ifndef __KERNEL__
+# error Userspace should not include obd_support.h.
+#endif /* !__KERNEL__ */
+
+#include <linux/atomic.h>
+#include <linux/highmem.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
#include <libcfs/libcfs.h>
-#include <lvfs.h>
#include <lprocfs_status.h>
-
-#if defined(__linux__)
-#include <linux/obd_support.h>
-#elif defined(__APPLE__)
-#include <darwin/obd_support.h>
-#elif defined(__WINNT__)
-#include <winnt/obd_support.h>
-#else
-#error Unsupported operating system.
-#endif
+#include <lustre_handles.h>
/* global variables */
extern struct lprocfs_stats *obd_memory;
enum {
OBD_MEMORY_STAT = 0,
- OBD_MEMORY_PAGES_STAT = 1,
OBD_STATS_NUM,
};
extern unsigned int ldlm_timeout; /* seconds */
extern unsigned int obd_timeout_set;
extern unsigned int ldlm_timeout_set;
+extern unsigned int bulk_timeout;
extern unsigned int at_min;
extern unsigned int at_max;
extern unsigned int at_history;
extern int at_early_margin;
extern int at_extra;
-extern unsigned int obd_sync_filter;
-extern unsigned int obd_max_dirty_pages;
-extern cfs_atomic_t obd_dirty_pages;
-extern cfs_atomic_t obd_dirty_transit_pages;
-extern unsigned int obd_alloc_fail_rate;
+extern unsigned long obd_max_dirty_pages;
+extern atomic_long_t obd_dirty_pages;
+extern atomic_long_t obd_dirty_transit_pages;
extern char obd_jobid_var[];
-/* lvfs.c */
-int obd_alloc_fail(const void *ptr, const char *name, const char *type,
- size_t size, const char *file, int line);
-
/* Some hash init argument constants */
#define HASH_POOLS_BKT_BITS 3
#define HASH_POOLS_CUR_BITS 3
#define HASH_NID_STATS_BKT_BITS 5
#define HASH_NID_STATS_CUR_BITS 7
#define HASH_NID_STATS_MAX_BITS 12
+#define HASH_GEN_BKT_BITS 5
+#define HASH_GEN_CUR_BITS 7
+#define HASH_GEN_MAX_BITS 12
#define HASH_LQE_BKT_BITS 5
#define HASH_LQE_CUR_BITS 7
#define HASH_LQE_MAX_BITS 12
#define OBD_FAIL_MDS_ALL_REQUEST_NET 0x123
#define OBD_FAIL_MDS_SYNC_NET 0x124
#define OBD_FAIL_MDS_SYNC_PACK 0x125
-#define OBD_FAIL_MDS_DONE_WRITING_NET 0x126
-#define OBD_FAIL_MDS_DONE_WRITING_PACK 0x127
+/* OBD_FAIL_MDS_DONE_WRITING_NET 0x126 obsolete since 2.8.0 */
+/* OBD_FAIL_MDS_DONE_WRITING_PACK 0x127 obsolete since 2.8.0 */
#define OBD_FAIL_MDS_ALLOC_OBDO 0x128
#define OBD_FAIL_MDS_PAUSE_OPEN 0x129
#define OBD_FAIL_MDS_STATFS_LCW_SLEEP 0x12a
#define OBD_FAIL_MDS_OPEN_CREATE 0x12b
#define OBD_FAIL_MDS_OST_SETATTR 0x12c
-#define OBD_FAIL_MDS_QUOTACHECK_NET 0x12d
+/* OBD_FAIL_MDS_QUOTACHECK_NET 0x12d obsolete since 2.4 */
#define OBD_FAIL_MDS_QUOTACTL_NET 0x12e
#define OBD_FAIL_MDS_CLIENT_ADD 0x12f
#define OBD_FAIL_MDS_GETXATTR_NET 0x130
#define OBD_FAIL_MDS_HSM_REQUEST_NET 0x14c
#define OBD_FAIL_MDS_HSM_CT_REGISTER_NET 0x14d
#define OBD_FAIL_MDS_HSM_CT_UNREGISTER_NET 0x14e
+#define OBD_FAIL_MDS_SWAP_LAYOUTS_NET 0x14f
#define OBD_FAIL_MDS_HSM_ACTION_NET 0x150
+#define OBD_FAIL_MDS_CHANGELOG_INIT 0x151
+#define OBD_FAIL_MDS_HSM_SWAP_LAYOUTS 0x152
+#define OBD_FAIL_MDS_RENAME 0x153
+#define OBD_FAIL_MDS_RENAME2 0x154
+#define OBD_FAIL_MDS_RENAME3 0x155
+#define OBD_FAIL_MDS_RENAME4 0x156
+#define OBD_FAIL_MDS_LDLM_REPLY_NET 0x157
+#define OBD_FAIL_MDS_STALE_DIR_LAYOUT 0x158
+#define OBD_FAIL_MDS_REINT_MULTI_NET 0x159
+#define OBD_FAIL_MDS_REINT_MULTI_NET_REP 0x15a
/* layout lock */
#define OBD_FAIL_MDS_NO_LL_GETATTR 0x170
#define OBD_FAIL_OSD_SCRUB_CRASH 0x191
#define OBD_FAIL_OSD_SCRUB_FATAL 0x192
#define OBD_FAIL_OSD_FID_MAPPING 0x193
+#define OBD_FAIL_OSD_LMA_INCOMPAT 0x194
+#define OBD_FAIL_OSD_COMPAT_INVALID_ENTRY 0x195
+#define OBD_FAIL_OSD_COMPAT_NO_ENTRY 0x196
+#define OBD_FAIL_OSD_OST_EA_FID_SET 0x197
#define OBD_FAIL_OST 0x200
#define OBD_FAIL_OST_CONNECT_NET 0x201
#define OBD_FAIL_OST_ENOSPC 0x215
#define OBD_FAIL_OST_EROFS 0x216
#define OBD_FAIL_OST_ENOENT 0x217
-#define OBD_FAIL_OST_QUOTACHECK_NET 0x218
+/* OBD_FAIL_OST_QUOTACHECK_NET 0x218 obsolete since 2.4 */
#define OBD_FAIL_OST_QUOTACTL_NET 0x219
#define OBD_FAIL_OST_CHECKSUM_RECEIVE 0x21a
#define OBD_FAIL_OST_CHECKSUM_SEND 0x21b
#define OBD_FAIL_OST_ENOINO 0x229
#define OBD_FAIL_OST_DQACQ_NET 0x230
#define OBD_FAIL_OST_STATFS_EINPROGRESS 0x231
+#define OBD_FAIL_OST_SET_INFO_NET 0x232
+#define OBD_FAIL_OST_NODESTROY 0x233
#define OBD_FAIL_LDLM 0x300
#define OBD_FAIL_LDLM_NAMESPACE_NEW 0x301
#define OBD_FAIL_LDLM_NEW_LOCK 0x319
#define OBD_FAIL_LDLM_AGL_DELAY 0x31a
#define OBD_FAIL_LDLM_AGL_NOLOCK 0x31b
+#define OBD_FAIL_LDLM_OST_LVB 0x31c
+#define OBD_FAIL_LDLM_ENQUEUE_HANG 0x31d
+#define OBD_FAIL_LDLM_BL_EVICT 0x31e
+#define OBD_FAIL_LDLM_PAUSE_CANCEL2 0x31f
+#define OBD_FAIL_LDLM_CP_CB_WAIT2 0x320
+#define OBD_FAIL_LDLM_CP_CB_WAIT3 0x321
+#define OBD_FAIL_LDLM_CP_CB_WAIT4 0x322
+#define OBD_FAIL_LDLM_CP_CB_WAIT5 0x323
+#define OBD_FAIL_LDLM_SRV_BL_AST 0x324
+#define OBD_FAIL_LDLM_SRV_CP_AST 0x325
+#define OBD_FAIL_LDLM_SRV_GL_AST 0x326
+#define OBD_FAIL_LDLM_WATERMARK_LOW 0x327
+#define OBD_FAIL_LDLM_WATERMARK_HIGH 0x328
/* LOCKLESS IO */
#define OBD_FAIL_LDLM_SET_CONTENTION 0x385
#define OBD_FAIL_PTLRPC_CLIENT_BULK_CB2 0x515
#define OBD_FAIL_PTLRPC_DELAY_IMP_FULL 0x516
#define OBD_FAIL_PTLRPC_CANCEL_RESEND 0x517
+#define OBD_FAIL_PTLRPC_CLIENT_BULK_CB3 0x520
#define OBD_FAIL_OBD_PING_NET 0x600
#define OBD_FAIL_OBD_LOG_CANCEL_NET 0x601
#define OBD_FAIL_OBD_LOGD_NET 0x602
-#define OBD_FAIL_OBD_QC_CALLBACK_NET 0x603
+/* OBD_FAIL_OBD_QC_CALLBACK_NET 0x603 obsolete since 2.4 */
#define OBD_FAIL_OBD_DQACQ 0x604
#define OBD_FAIL_OBD_LLOG_SETUP 0x605
#define OBD_FAIL_OBD_LOG_CANCEL_REP 0x606
#define OBD_FAIL_OBD_IDX_READ_NET 0x607
#define OBD_FAIL_OBD_IDX_READ_BREAK 0x608
+#define OBD_FAIL_OBD_NO_LRU 0x609
#define OBD_FAIL_TGT_REPLY_NET 0x700
#define OBD_FAIL_TGT_CONN_RACE 0x701
#define OBD_FAIL_TGT_REPLAY_DROP 0x707
#define OBD_FAIL_TGT_FAKE_EXP 0x708
#define OBD_FAIL_TGT_REPLAY_DELAY 0x709
-#define OBD_FAIL_TGT_LAST_REPLAY 0x710
+/* #define OBD_FAIL_TGT_LAST_REPLAY 0x710 (obsoleted) */
#define OBD_FAIL_TGT_CLIENT_ADD 0x711
#define OBD_FAIL_TGT_RCVG_FLAG 0x712
+#define OBD_FAIL_TGT_DELAY_CONDITIONAL 0x713
+#define OBD_FAIL_TGT_REPLAY_DELAY2 0x714
+#define OBD_FAIL_TGT_REPLAY_RECONNECT 0x715
#define OBD_FAIL_MDC_REVALIDATE_PAUSE 0x800
#define OBD_FAIL_MDC_ENQUEUE_PAUSE 0x801
#define OBD_FAIL_MGC_PAUSE_PROCESS_LOG 0x903
#define OBD_FAIL_MGS_PAUSE_REQ 0x904
#define OBD_FAIL_MGS_PAUSE_TARGET_REG 0x905
+#define OBD_FAIL_MGS_CONNECT_NET 0x906
+#define OBD_FAIL_MGS_DISCONNECT_NET 0x907
+#define OBD_FAIL_MGS_SET_INFO_NET 0x908
+#define OBD_FAIL_MGS_EXCEPTION_NET 0x909
+#define OBD_FAIL_MGS_TARGET_REG_NET 0x90a
+#define OBD_FAIL_MGS_TARGET_DEL_NET 0x90b
+#define OBD_FAIL_MGS_CONFIG_READ_NET 0x90c
+#define OBD_FAIL_MGS_LDLM_REPLY_NET 0x90d
#define OBD_FAIL_QUOTA_DQACQ_NET 0xA01
#define OBD_FAIL_QUOTA_EDQUOT 0xA02
#define OBD_FAIL_LPROC_REMOVE 0xB00
-#define OBD_FAIL_GENERAL_ALLOC 0xC00
-
#define OBD_FAIL_SEQ 0x1000
#define OBD_FAIL_SEQ_QUERY_NET 0x1001
#define OBD_FAIL_SEQ_EXHAUST 0x1002
#define OBD_FAIL_FLD 0x1100
#define OBD_FAIL_FLD_QUERY_NET 0x1101
+#define OBD_FAIL_FLD_READ_NET 0x1102
#define OBD_FAIL_SEC_CTX 0x1200
#define OBD_FAIL_SEC_CTX_INIT_NET 0x1201
#define OBD_FAIL_LOCK_STATE_WAIT_INTR 0x1402
#define OBD_FAIL_LOV_INIT 0x1403
#define OBD_FAIL_GLIMPSE_DELAY 0x1404
-
-#define OBD_FAIL_UPDATE_OBJ_NET 0x1500
+#define OBD_FAIL_LLITE_XATTR_ENOMEM 0x1405
+#define OBD_FAIL_MAKE_LOVEA_HOLE 0x1406
+#define OBD_FAIL_LLITE_LOST_LAYOUT 0x1407
+
+#define OBD_FAIL_FID_INDIR 0x1501
+#define OBD_FAIL_FID_INLMA 0x1502
+#define OBD_FAIL_FID_IGIF 0x1504
+#define OBD_FAIL_FID_LOOKUP 0x1505
+#define OBD_FAIL_FID_NOLMA 0x1506
+
+/* LFSCK */
+#define OBD_FAIL_LFSCK_DELAY1 0x1600
+#define OBD_FAIL_LFSCK_DELAY2 0x1601
+#define OBD_FAIL_LFSCK_DELAY3 0x1602
+#define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
+#define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
+#define OBD_FAIL_LFSCK_LINKEA_MORE2 0x1605
+#define OBD_FAIL_LFSCK_FATAL1 0x1608
+#define OBD_FAIL_LFSCK_FATAL2 0x1609
+#define OBD_FAIL_LFSCK_CRASH 0x160a
+#define OBD_FAIL_LFSCK_NO_AUTO 0x160b
+#define OBD_FAIL_LFSCK_NO_DOUBLESCAN 0x160c
+#define OBD_FAIL_LFSCK_SKIP_LASTID 0x160d
+#define OBD_FAIL_LFSCK_DELAY4 0x160e
+#define OBD_FAIL_LFSCK_BAD_LMMOI 0x160f
+#define OBD_FAIL_LFSCK_DANGLING 0x1610
+#define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
+#define OBD_FAIL_LFSCK_UNMATCHED_PAIR2 0x1612
+#define OBD_FAIL_LFSCK_BAD_OWNER 0x1613
+#define OBD_FAIL_LFSCK_MULTIPLE_REF 0x1614
+#define OBD_FAIL_LFSCK_LOST_STRIPE 0x1615
+#define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
+#define OBD_FAIL_LFSCK_NOPFID 0x1617
+#define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
+#define OBD_FAIL_LFSCK_INVALID_PFID 0x1619
+#define OBD_FAIL_LFSCK_LOST_SPEOBJ 0x161a
+#define OBD_FAIL_LFSCK_DELAY5 0x161b
+#define OBD_FAIL_LFSCK_BAD_NETWORK 0x161c
+#define OBD_FAIL_LFSCK_NO_LINKEA 0x161d
+#define OBD_FAIL_LFSCK_BAD_PARENT 0x161e
+#define OBD_FAIL_LFSCK_DANGLING2 0x1620
+#define OBD_FAIL_LFSCK_DANGLING3 0x1621
+#define OBD_FAIL_LFSCK_MUL_REF 0x1622
+#define OBD_FAIL_LFSCK_BAD_TYPE 0x1623
+#define OBD_FAIL_LFSCK_NO_NAMEENTRY 0x1624
+#define OBD_FAIL_LFSCK_MORE_NLINK 0x1625
+#define OBD_FAIL_LFSCK_LESS_NLINK 0x1626
+#define OBD_FAIL_LFSCK_LINKEA_OVERFLOW 0x1627
+#define OBD_FAIL_LFSCK_BAD_NAME_HASH 0x1628
+#define OBD_FAIL_LFSCK_LOST_MASTER_LMV 0x1629
+#define OBD_FAIL_LFSCK_LOST_SLAVE_LMV 0x162a
+#define OBD_FAIL_LFSCK_BAD_SLAVE_LMV 0x162b
+#define OBD_FAIL_LFSCK_BAD_SLAVE_NAME 0x162c
+
+#define OBD_FAIL_LFSCK_NOTIFY_NET 0x16f0
+#define OBD_FAIL_LFSCK_QUERY_NET 0x16f1
+
+/* UPDATE */
+#define OBD_FAIL_OUT_UPDATE_NET 0x1700
+#define OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
+#define OBD_FAIL_SPLIT_UPDATE_REC 0x1702
+#define OBD_FAIL_LARGE_STRIPE 0x1703
+
+/* MIGRATE */
+#define OBD_FAIL_MIGRATE_NET_REP 0x1800
+#define OBD_FAIL_MIGRATE_ENTRIES 0x1801
+#define OBD_FAIL_MIGRATE_LINKEA 0x1802
+#define OBD_FAIL_MIGRATE_DELAY 0x1803
+
+/* LMV */
+#define OBD_FAIL_UNKNOWN_LMV_STRIPE 0x1901
+
+/* DT */
+#define OBD_FAIL_DT_DECLARE_ATTR_GET 0x2000
+#define OBD_FAIL_DT_ATTR_GET 0x2001
+#define OBD_FAIL_DT_DECLARE_ATTR_SET 0x2002
+#define OBD_FAIL_DT_ATTR_SET 0x2003
+#define OBD_FAIL_DT_DECLARE_XATTR_GET 0x2004
+#define OBD_FAIL_DT_XATTR_GET 0x2005
+#define OBD_FAIL_DT_DECLARE_XATTR_SET 0x2006
+#define OBD_FAIL_DT_XATTR_SET 0x2007
+#define OBD_FAIL_DT_DECLARE_XATTR_DEL 0x2008
+#define OBD_FAIL_DT_XATTR_DEL 0x2009
+#define OBD_FAIL_DT_XATTR_LIST 0x200a
+#define OBD_FAIL_DT_DECLARE_CREATE 0x200b
+#define OBD_FAIL_DT_CREATE 0x200c
+#define OBD_FAIL_DT_DECLARE_DESTROY 0x200d
+#define OBD_FAIL_DT_DESTROY 0x200e
+#define OBD_FAIL_DT_INDEX_TRY 0x200f
+#define OBD_FAIL_DT_DECLARE_REF_ADD 0x2010
+#define OBD_FAIL_DT_REF_ADD 0x2011
+#define OBD_FAIL_DT_DECLARE_REF_DEL 0x2012
+#define OBD_FAIL_DT_REF_DEL 0x2013
+#define OBD_FAIL_DT_DECLARE_INSERT 0x2014
+#define OBD_FAIL_DT_INSERT 0x2015
+#define OBD_FAIL_DT_DECLARE_DELETE 0x2016
+#define OBD_FAIL_DT_DELETE 0x2017
+#define OBD_FAIL_DT_LOOKUP 0x2018
/* Assign references to moved code to reduce code changes */
#define OBD_FAIL_PRECHECK(id) CFS_FAIL_PRECHECK(id)
#define OBD_FAIL_ONCE CFS_FAIL_ONCE
#define OBD_FAILED CFS_FAILED
-extern cfs_atomic_t libcfs_kmemory;
+#define LUT_FAIL_CLASS(fail_id) (((fail_id) >> 8) << 16)
+#define LUT_FAIL_MGT LUT_FAIL_CLASS(OBD_FAIL_MGS)
+#define LUT_FAIL_MDT LUT_FAIL_CLASS(OBD_FAIL_MDS)
+#define LUT_FAIL_OST LUT_FAIL_CLASS(OBD_FAIL_OST)
+
+extern atomic_t libcfs_kmemory;
-#ifdef LPROCFS
+#ifdef CONFIG_PROC_FS
#define obd_memory_add(size) \
lprocfs_counter_add(obd_memory, OBD_MEMORY_STAT, (long)(size))
#define obd_memory_sub(size) \
#define obd_memory_sum() \
lprocfs_stats_collector(obd_memory, OBD_MEMORY_STAT, \
LPROCFS_FIELDS_FLAGS_SUM)
-#define obd_pages_add(order) \
- lprocfs_counter_add(obd_memory, OBD_MEMORY_PAGES_STAT, \
- (long)(1 << (order)))
-#define obd_pages_sub(order) \
- lprocfs_counter_sub(obd_memory, OBD_MEMORY_PAGES_STAT, \
- (long)(1 << (order)))
-#define obd_pages_sum() \
- lprocfs_stats_collector(obd_memory, OBD_MEMORY_PAGES_STAT, \
- LPROCFS_FIELDS_FLAGS_SUM)
extern void obd_update_maxusage(void);
extern __u64 obd_memory_max(void);
-extern __u64 obd_pages_max(void);
-#else
+#else /* CONFIG_PROC_FS */
extern __u64 obd_alloc;
-extern __u64 obd_pages;
extern __u64 obd_max_alloc;
-extern __u64 obd_max_pages;
static inline void obd_memory_add(long size)
{
obd_alloc -= size;
}
-static inline void obd_pages_add(int order)
-{
- obd_pages += 1<< order;
- if (obd_pages > obd_max_pages)
- obd_max_pages = obd_pages;
-}
-
-static inline void obd_pages_sub(int order)
-{
- obd_pages -= 1<< order;
-}
-
#define obd_memory_sum() (obd_alloc)
-#define obd_pages_sum() (obd_pages)
#define obd_memory_max() (obd_max_alloc)
-#define obd_pages_max() (obd_max_pages)
-#endif
+#endif /* !CONFIG_PROC_FS */
#define OBD_DEBUG_MEMUSAGE (1)
#endif /* !OBD_DEBUG_MEMUSAGE */
-#ifdef RANDOM_FAIL_ALLOC
-#define HAS_FAIL_ALLOC_FLAG OBD_FAIL_CHECK(OBD_FAIL_GENERAL_ALLOC)
-#else
-#define HAS_FAIL_ALLOC_FLAG 0
-#endif
-
-#define OBD_ALLOC_FAIL_BITS 24
-#define OBD_ALLOC_FAIL_MASK ((1 << OBD_ALLOC_FAIL_BITS) - 1)
-#define OBD_ALLOC_FAIL_MULT (OBD_ALLOC_FAIL_MASK / 100)
-
-#if defined(LUSTRE_UTILS) /* this version is for utils only */
#define __OBD_MALLOC_VERBOSE(ptr, cptab, cpt, size, flags) \
do { \
(ptr) = (cptab) == NULL ? \
- cfs_alloc(size, flags) : \
- cfs_cpt_malloc(cptab, cpt, size, flags); \
- if (unlikely((ptr) == NULL)) { \
- CERROR("kmalloc of '" #ptr "' (%d bytes) failed at %s:%d\n", \
- (int)(size), __FILE__, __LINE__); \
- } else { \
- memset(ptr, 0, size); \
- CDEBUG(D_MALLOC, "kmalloced '" #ptr "': %d at %p\n", \
- (int)(size), ptr); \
- } \
+ kmalloc(size, flags | __GFP_ZERO) : \
+ cfs_cpt_malloc(cptab, cpt, size, flags | __GFP_ZERO); \
+ if (likely((ptr) != NULL)) \
+ OBD_ALLOC_POST(ptr, size, "kmalloced"); \
} while (0)
-#else /* this version is for the kernel and liblustre */
-#define OBD_FREE_RTN0(ptr) \
-({ \
- cfs_free(ptr); \
- (ptr) = NULL; \
- 0; \
-})
-
-#define __OBD_MALLOC_VERBOSE(ptr, cptab, cpt, size, flags) \
-do { \
- (ptr) = (cptab) == NULL ? \
- cfs_alloc(size, flags) : \
- cfs_cpt_malloc(cptab, cpt, size, flags); \
- if (likely((ptr) != NULL && \
- (!HAS_FAIL_ALLOC_FLAG || obd_alloc_fail_rate == 0 || \
- !obd_alloc_fail(ptr, #ptr, "km", size, \
- __FILE__, __LINE__) || \
- OBD_FREE_RTN0(ptr)))){ \
- memset(ptr, 0, size); \
- OBD_ALLOC_POST(ptr, size, "kmalloced"); \
- } \
-} while (0)
-#endif
-
#define OBD_ALLOC_GFP(ptr, size, gfp_mask) \
__OBD_MALLOC_VERBOSE(ptr, NULL, 0, size, gfp_mask)
-#define OBD_ALLOC(ptr, size) OBD_ALLOC_GFP(ptr, size, CFS_ALLOC_IO)
-#define OBD_ALLOC_WAIT(ptr, size) OBD_ALLOC_GFP(ptr, size, CFS_ALLOC_STD)
+#define OBD_ALLOC(ptr, size) OBD_ALLOC_GFP(ptr, size, GFP_NOFS)
+#define OBD_ALLOC_WAIT(ptr, size) OBD_ALLOC_GFP(ptr, size, GFP_KERNEL)
#define OBD_ALLOC_PTR(ptr) OBD_ALLOC(ptr, sizeof *(ptr))
#define OBD_ALLOC_PTR_WAIT(ptr) OBD_ALLOC_WAIT(ptr, sizeof *(ptr))
__OBD_MALLOC_VERBOSE(ptr, cptab, cpt, size, gfp_mask)
#define OBD_CPT_ALLOC(ptr, cptab, cpt, size) \
- OBD_CPT_ALLOC_GFP(ptr, cptab, cpt, size, CFS_ALLOC_IO)
+ OBD_CPT_ALLOC_GFP(ptr, cptab, cpt, size, GFP_NOFS)
#define OBD_CPT_ALLOC_PTR(ptr, cptab, cpt) \
OBD_CPT_ALLOC(ptr, cptab, cpt, sizeof *(ptr))
-#ifdef __arch_um__
-
-# define OBD_VMALLOC(ptr, size) \
- OBD_ALLOC(ptr, size)
-# define OBD_CPT_VMALLOC(ptr, cptab, cpt, size) \
- OBD_CPT_ALLOC(ptr, cptab, cpt, size)
-
-#else /* !__arch_um__ */
-
-# define __OBD_VMALLOC_VEROBSE(ptr, cptab, cpt, size) \
+/* Direct use of __vmalloc() allows for protection flag specification
+ * (and particularly to not set __GFP_FS, which is likely to cause some
+ * deadlock situations in our code).
+ */
+#define __OBD_VMALLOC_VERBOSE(ptr, cptab, cpt, size) \
do { \
(ptr) = cptab == NULL ? \
- cfs_alloc_large(size) : \
- cfs_cpt_vmalloc(cptab, cpt, size); \
- if (unlikely((ptr) == NULL)) { \
- CERROR("vmalloc of '" #ptr "' (%d bytes) failed\n", \
- (int)(size)); \
- CERROR(LPU64" total bytes allocated by Lustre, %d by LNET\n", \
- obd_memory_sum(), cfs_atomic_read(&libcfs_kmemory)); \
- } else { \
- memset(ptr, 0, size); \
- OBD_ALLOC_POST(ptr, size, "vmalloced"); \
- } \
+ __vmalloc(size, GFP_NOFS | __GFP_HIGHMEM | __GFP_ZERO, \
+ PAGE_KERNEL) : \
+ cfs_cpt_vzalloc(cptab, cpt, size); \
+ if (unlikely((ptr) == NULL)) { \
+ CERROR("vmalloc of '" #ptr "' (%d bytes) failed\n", \
+ (int)(size)); \
+ CERROR(LPU64" total bytes allocated by Lustre, %d by LNET\n", \
+ obd_memory_sum(), atomic_read(&libcfs_kmemory)); \
+ } else { \
+ OBD_ALLOC_POST(ptr, size, "vmalloced"); \
+ } \
} while(0)
-# define OBD_VMALLOC(ptr, size) \
- __OBD_VMALLOC_VEROBSE(ptr, NULL, 0, size)
-# define OBD_CPT_VMALLOC(ptr, cptab, cpt, size) \
- __OBD_VMALLOC_VEROBSE(ptr, cptab, cpt, size)
-
-#endif
-
-#ifdef __KERNEL__
-
-/* Allocations above this size are considered too big and could not be done
- * atomically.
- *
- * Be very careful when changing this value, especially when decreasing it,
- * since vmalloc in Linux doesn't perform well on multi-cores system, calling
- * vmalloc in critical path would hurt peformance badly. See LU-66.
- */
-#define OBD_ALLOC_BIG (4 * CFS_PAGE_SIZE)
+#define OBD_VMALLOC(ptr, size) \
+ __OBD_VMALLOC_VERBOSE(ptr, NULL, 0, size)
+#define OBD_CPT_VMALLOC(ptr, cptab, cpt, size) \
+ __OBD_VMALLOC_VERBOSE(ptr, cptab, cpt, size)
#define OBD_ALLOC_LARGE(ptr, size) \
do { \
- if (size > OBD_ALLOC_BIG) \
+ OBD_ALLOC_GFP(ptr, size, GFP_NOFS | __GFP_NOWARN); \
+ if (ptr == NULL) \
OBD_VMALLOC(ptr, size); \
- else \
- OBD_ALLOC(ptr, size); \
} while (0)
#define OBD_CPT_ALLOC_LARGE(ptr, cptab, cpt, size) \
do { \
- if (size > OBD_ALLOC_BIG) \
+ OBD_CPT_ALLOC_GFP(ptr, cptab, cpt, size, GFP_NOFS | __GFP_NOWARN); \
+ if (ptr == NULL) \
OBD_CPT_VMALLOC(ptr, cptab, cpt, size); \
- else \
- OBD_CPT_ALLOC(ptr, cptab, cpt, size); \
-} while (0)
-
-#define OBD_FREE_LARGE(ptr, size) \
-do { \
- if (size > OBD_ALLOC_BIG) \
- OBD_VFREE(ptr, size); \
- else \
- OBD_FREE(ptr, size); \
} while (0)
-#else /* !__KERNEL__ */
-
-#define OBD_ALLOC_LARGE(ptr, size) \
- OBD_ALLOC(ptr, size)
-#define OBD_CPT_ALLOC_LARGE(ptr, cptab, cpt, size) \
- OBD_ALLOC(ptr, size)
-#define OBD_FREE_LARGE(ptr, size) \
- OBD_FREE(ptr, size)
-
-#endif /* __KERNEL__ */
+#define OBD_FREE_LARGE(ptr, size) OBD_FREE(ptr, size)
#ifdef CONFIG_DEBUG_SLAB
#define POISON(ptr, c, s) do {} while (0)
#endif
#ifdef POISON_BULK
-#define POISON_PAGE(page, val) do { memset(kmap(page), val, CFS_PAGE_SIZE); \
+#define POISON_PAGE(page, val) do { memset(kmap(page), val, PAGE_CACHE_SIZE); \
kunmap(page); } while (0)
#else
#define POISON_PAGE(page, val) do { } while (0)
#endif
-#ifdef __KERNEL__
#define OBD_FREE(ptr, size) \
do { \
- OBD_FREE_PRE(ptr, size, "kfreed"); \
- cfs_free(ptr); \
- POISON_PTR(ptr); \
+ if (is_vmalloc_addr(ptr)) { \
+ OBD_FREE_PRE(ptr, size, "vfreed"); \
+ vfree(ptr); \
+ } else { \
+ OBD_FREE_PRE(ptr, size, "kfreed"); \
+ kfree(ptr); \
+ } \
+ POISON_PTR(ptr); \
} while(0)
-
#define OBD_FREE_RCU(ptr, size, handle) \
do { \
struct portals_handle *__h = (handle); \
POISON_PTR(ptr); \
} while(0)
-#else
-#define OBD_FREE(ptr, size) ((void)(size), free((ptr)))
-#define OBD_FREE_RCU(ptr, size, handle) (OBD_FREE(ptr, size))
-#endif /* ifdef __KERNEL__ */
-
-#ifdef __arch_um__
-# define OBD_VFREE(ptr, size) OBD_FREE(ptr, size)
-#else
-# define OBD_VFREE(ptr, size) \
-do { \
- OBD_FREE_PRE(ptr, size, "vfreed"); \
- cfs_free_large(ptr); \
- POISON_PTR(ptr); \
-} while(0)
-#endif
-
/* we memset() the slab object to 0 when allocation succeeds, so DO NOT
* HAVE A CTOR THAT DOES ANYTHING. its work will be cleared here. we'd
* love to assert on that, but slab.c keeps kmem_cache_s all to itself. */
#define OBD_SLAB_FREE_RTN0(ptr, slab) \
({ \
- cfs_mem_cache_free((slab), (ptr)); \
+ kmem_cache_free((slab), (ptr)); \
(ptr) = NULL; \
0; \
})
#define __OBD_SLAB_ALLOC_VERBOSE(ptr, slab, cptab, cpt, size, type) \
do { \
- LASSERT(ergo(type != CFS_ALLOC_ATOMIC, !cfs_in_interrupt())); \
+ LASSERT(ergo((type) != GFP_ATOMIC, !in_interrupt())); \
(ptr) = (cptab) == NULL ? \
- cfs_mem_cache_alloc(slab, type) : \
- cfs_mem_cache_cpt_alloc(slab, cptab, cpt, type); \
- if (likely((ptr) != NULL && \
- (!HAS_FAIL_ALLOC_FLAG || obd_alloc_fail_rate == 0 || \
- !obd_alloc_fail(ptr, #ptr, "slab-", size, \
- __FILE__, __LINE__) || \
- OBD_SLAB_FREE_RTN0(ptr, slab)))) { \
- memset(ptr, 0, size); \
- OBD_ALLOC_POST(ptr, size, "slab-alloced"); \
- } \
+ kmem_cache_alloc(slab, type | __GFP_ZERO) : \
+ cfs_mem_cache_cpt_alloc(slab, cptab, cpt, type | __GFP_ZERO); \
+ if (likely((ptr))) \
+ OBD_ALLOC_POST(ptr, size, "slab-alloced"); \
} while(0)
#define OBD_SLAB_ALLOC_GFP(ptr, slab, size, flags) \
#define OBD_SLAB_FREE(ptr, slab, size) \
do { \
OBD_FREE_PRE(ptr, size, "slab-freed"); \
- cfs_mem_cache_free(slab, ptr); \
+ kmem_cache_free(slab, ptr); \
POISON_PTR(ptr); \
} while(0)
#define OBD_SLAB_ALLOC(ptr, slab, size) \
- OBD_SLAB_ALLOC_GFP(ptr, slab, size, CFS_ALLOC_IO)
+ OBD_SLAB_ALLOC_GFP(ptr, slab, size, GFP_NOFS)
#define OBD_SLAB_CPT_ALLOC(ptr, slab, cptab, cpt, size) \
- OBD_SLAB_CPT_ALLOC_GFP(ptr, slab, cptab, cpt, size, CFS_ALLOC_IO)
+ OBD_SLAB_CPT_ALLOC_GFP(ptr, slab, cptab, cpt, size, GFP_NOFS)
#define OBD_SLAB_ALLOC_PTR(ptr, slab) \
OBD_SLAB_ALLOC(ptr, slab, sizeof *(ptr))
#define OBD_SLAB_ALLOC_PTR_GFP(ptr, slab, flags) \
OBD_SLAB_ALLOC_GFP(ptr, slab, sizeof *(ptr), flags)
-#define OBD_SLAB_CPT_ALLOC_PTR_GFP(ptr, slab, ctab, cpt, flags) \
+#define OBD_SLAB_CPT_ALLOC_PTR_GFP(ptr, slab, cptab, cpt, flags) \
OBD_SLAB_CPT_ALLOC_GFP(ptr, slab, cptab, cpt, sizeof *(ptr), flags)
#define OBD_SLAB_FREE_PTR(ptr, slab) \
#define KEY_IS(str) \
(keylen >= (sizeof(str)-1) && memcmp(key, str, (sizeof(str)-1)) == 0)
-/* Wrapper for contiguous page frame allocation */
-#define __OBD_PAGE_ALLOC_VERBOSE(ptr, cptab, cpt, gfp_mask) \
-do { \
- (ptr) = (cptab) == NULL ? \
- cfs_alloc_page(gfp_mask) : \
- cfs_page_cpt_alloc(cptab, cpt, gfp_mask); \
- if (unlikely((ptr) == NULL)) { \
- CERROR("alloc_pages of '" #ptr "' %d page(s) / "LPU64" bytes "\
- "failed\n", (int)1, \
- (__u64)(1 << CFS_PAGE_SHIFT)); \
- CERROR(LPU64" total bytes and "LPU64" total pages " \
- "("LPU64" bytes) allocated by Lustre, " \
- "%d total bytes by LNET\n", \
- obd_memory_sum(), \
- obd_pages_sum() << CFS_PAGE_SHIFT, \
- obd_pages_sum(), \
- cfs_atomic_read(&libcfs_kmemory)); \
- } else { \
- obd_pages_add(0); \
- CDEBUG(D_MALLOC, "alloc_pages '" #ptr "': %d page(s) / " \
- LPU64" bytes at %p.\n", \
- (int)1, \
- (__u64)(1 << CFS_PAGE_SHIFT), ptr); \
- } \
-} while (0)
-
-#define OBD_PAGE_ALLOC(ptr, gfp_mask) \
- __OBD_PAGE_ALLOC_VERBOSE(ptr, NULL, 0, gfp_mask)
-#define OBD_PAGE_CPT_ALLOC(ptr, cptab, cpt, gfp_mask) \
- __OBD_PAGE_ALLOC_VERBOSE(ptr, cptab, cpt, gfp_mask)
-
-#define OBD_PAGE_FREE(ptr) \
-do { \
- LASSERT(ptr); \
- obd_pages_sub(0); \
- CDEBUG(D_MALLOC, "free_pages '" #ptr "': %d page(s) / "LPU64" bytes " \
- "at %p.\n", \
- (int)1, (__u64)(1 << CFS_PAGE_SHIFT), \
- ptr); \
- cfs_free_page(ptr); \
- (ptr) = (void *)0xdeadbeef; \
-} while (0)
-
#endif