X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Finclude%2Fobd_cksum.h;h=d546bb8447c0a5c7571bd97aefceb54215d885e3;hp=7166f7ab4db6db9e770791525a257df215765e3e;hb=84a3fd67356c8073a917ea6abd63928055e38156;hpb=220c6d92200792c318f5248e8e44dbcfc712c788 diff --git a/lustre/include/obd_cksum.h b/lustre/include/obd_cksum.h index 7166f7a..d546bb8 100644 --- a/lustre/include/obd_cksum.h +++ b/lustre/include/obd_cksum.h @@ -34,157 +34,23 @@ #ifndef __OBD_CKSUM #define __OBD_CKSUM - -#if defined(__linux__) -#include -#elif defined(__APPLE__) -#include -#elif defined(__WINNT__) -#include -#else -#error Unsupported operating system. -#endif - +#include #include -/* - * Checksums - */ - -#ifndef HAVE_ARCH_CRC32 -/* crc32_le lifted from the Linux kernel, which had the following to say: - * - * This code is in the public domain; copyright abandoned. - * Liability for non-performance of this code is limited to the amount - * you paid for it. Since it is distributed for free, your refund will - * be very very small. If it breaks, you get to keep both pieces. - */ -#define CRCPOLY_LE 0xedb88320 -/** - * crc32_le() - Calculate bitwise little-endian Ethernet AUTODIN II CRC32 - * \param crc seed value for computation. ~0 for Ethernet, sometimes 0 for - * other uses, or the previous crc32 value if computing incrementally. - * \param p - pointer to buffer over which CRC is run - * \param len- length of buffer \a p - */ -static inline __u32 crc32_le(__u32 crc, unsigned char const *p, size_t len) -{ - int i; - while (len--) { - crc ^= *p++; - for (i = 0; i < 8; i++) - crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY_LE : 0); - } - return crc; -} -#endif - -#ifdef HAVE_ADLER -/* Adler-32 is supported */ -#define CHECKSUM_ADLER OBD_CKSUM_ADLER -#else -#define CHECKSUM_ADLER 0 -#endif - -#ifdef X86_FEATURE_XMM4_2 -/* Call Nehalem+ CRC32C harware acceleration instruction on individual bytes. */ -static inline __u32 crc32c_hw_byte(__u32 crc, unsigned char const *p, - size_t bytes) +static inline unsigned char cksum_obd2cfs(cksum_type_t cksum_type) { - while (bytes--) { - __asm__ __volatile__ ( - ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1" - : "=S"(crc) - : "0"(crc), "c"(*p) - ); - p++; - } - - return crc; -} - -#if BITS_PER_LONG > 32 -#define WORD_SHIFT 3 -#define WORD_MASK 7 -#define REX "0x48, " -#else -#define WORD_SHIFT 2 -#define WORD_MASK 3 -#define REX "" -#endif - -/* Do we need to worry about unaligned input data here? */ -static inline __u32 crc32c_hw(__u32 crc, unsigned char const *p, size_t len) -{ - unsigned int words = len >> WORD_SHIFT; - unsigned int bytes = len & WORD_MASK; - long *ptmp = (long *)p; - - while (words--) { - __asm__ __volatile__( - ".byte 0xf2, " REX "0xf, 0x38, 0xf1, 0xf1;" - : "=S"(crc) - : "0"(crc), "c"(*ptmp) - ); - ptmp++; - } - - if (bytes) - crc = crc32c_hw_byte(crc, (unsigned char *)ptmp, bytes); - - return crc; -} -#else -/* We should never call this unless the CPU has previously been detected to - * support this instruction in the SSE4.2 feature set. b=23549 */ -static inline __u32 crc32c_hw(__u32 crc, unsigned char const *p,size_t len) -{ - LBUG(); -} -#endif - -static inline __u32 init_checksum(cksum_type_t cksum_type) -{ - switch(cksum_type) { - case OBD_CKSUM_CRC32C: - return ~0U; -#ifdef HAVE_ADLER - case OBD_CKSUM_ADLER: - return 1U; -#endif - case OBD_CKSUM_CRC32: - return ~0U; - default: - CERROR("Unknown checksum type (%x)!!!\n", cksum_type); - LBUG(); - } - return 0; -} - -static inline __u32 fini_checksum(__u32 cksum, cksum_type_t cksum_type) -{ - if (cksum_type == OBD_CKSUM_CRC32C) - return ~cksum; - return cksum; -} - -static inline __u32 compute_checksum(__u32 cksum, unsigned char const *p, - size_t len, cksum_type_t cksum_type) -{ - switch(cksum_type) { - case OBD_CKSUM_CRC32C: - return crc32c_hw(cksum, p, len); -#ifdef HAVE_ADLER - case OBD_CKSUM_ADLER: - return adler32(cksum, p, len); -#endif - case OBD_CKSUM_CRC32: - return crc32_le(cksum, p, len); - default: - CERROR("Unknown checksum type (%x)!!!\n", cksum_type); - LBUG(); - } - return 0; + switch (cksum_type) { + case OBD_CKSUM_CRC32: + return CFS_HASH_ALG_CRC32; + case OBD_CKSUM_ADLER: + return CFS_HASH_ALG_ADLER32; + case OBD_CKSUM_CRC32C: + return CFS_HASH_ALG_CRC32C; + default: + CERROR("Unknown checksum type (%x)!!!\n", cksum_type); + LBUG(); + } + return 0; } /* The OBD_FL_CKSUM_* flags is packed into 5 bits of o_flags, since there can @@ -194,79 +60,113 @@ static inline __u32 compute_checksum(__u32 cksum, unsigned char const *p, * since they need to represent the full range of checksum algorithms that * both the client and server can understand. * - * In case of an unsupported types/flags we fall back to CRC32 (even though - * it isn't very fast) because that is supported by all clients - * checksums, since 1.6.5 (or earlier via patches). + * In case of an unsupported types/flags we fall back to ADLER + * because that is supported by all clients since 1.8 * - * These flags should be listed in order of descending performance, so that - * in case multiple algorithms are supported the best one is used. */ + * In case multiple algorithms are supported the best one is used. */ static inline obd_flag cksum_type_pack(cksum_type_t cksum_type) { - if (cksum_type & OBD_CKSUM_CRC32C) - return OBD_FL_CKSUM_CRC32C; -#ifdef HAVE_ADLER - if (cksum_type & OBD_CKSUM_ADLER) - return OBD_FL_CKSUM_ADLER; -#endif - if (unlikely(cksum_type && !(cksum_type & OBD_CKSUM_CRC32))) - CWARN("unknown cksum type %x\n", cksum_type); - - return OBD_FL_CKSUM_CRC32; + unsigned int performance = 0, tmp; + obd_flag flag = OBD_FL_CKSUM_ADLER; + + if (cksum_type & OBD_CKSUM_CRC32) { + tmp = cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_CRC32)); + if (tmp > performance) { + performance = tmp; + flag = OBD_FL_CKSUM_CRC32; + } + } + if (cksum_type & OBD_CKSUM_CRC32C) { + tmp = cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_CRC32C)); + if (tmp > performance) { + performance = tmp; + flag = OBD_FL_CKSUM_CRC32C; + } + } + if (cksum_type & OBD_CKSUM_ADLER) { + tmp = cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_ADLER)); + if (tmp > performance) { + performance = tmp; + flag = OBD_FL_CKSUM_ADLER; + } + } + if (unlikely(cksum_type && !(cksum_type & (OBD_CKSUM_CRC32C | + OBD_CKSUM_CRC32 | + OBD_CKSUM_ADLER)))) + CWARN("unknown cksum type %x\n", cksum_type); + + return flag; } static inline cksum_type_t cksum_type_unpack(obd_flag o_flags) { - switch (o_flags & OBD_FL_CKSUM_ALL) { - case OBD_FL_CKSUM_CRC32C: - return OBD_CKSUM_CRC32C; - case OBD_FL_CKSUM_ADLER: -#ifdef HAVE_ADLER - return OBD_CKSUM_ADLER; -#else - CWARN("checksum type is set to adler32, but adler32 is not " - "supported (%x)\n", o_flags); - break; -#endif - default: - break; - } - - /* 1.6.4- only supported CRC32 and didn't set o_flags */ - return OBD_CKSUM_CRC32; + switch (o_flags & OBD_FL_CKSUM_ALL) { + case OBD_FL_CKSUM_CRC32C: + return OBD_CKSUM_CRC32C; + case OBD_FL_CKSUM_CRC32: + return OBD_CKSUM_CRC32; + default: + break; + } + + return OBD_CKSUM_ADLER; } /* Return a bitmask of the checksum types supported on this system. - * - * CRC32 is a required for compatibility (starting with 1.6.5), - * after which we could move to Adler as the base checksum type. - * - * If hardware crc32c support is not available, it is slower than Adler, - * so don't include it, even if it could be emulated in software. b=23549 */ -static inline cksum_type_t cksum_types_supported(void) + * 1.8 supported ADLER it is base and not depend on hw + * Client uses all available local algos + */ +static inline cksum_type_t cksum_types_supported_client(void) { - cksum_type_t ret = OBD_CKSUM_CRC32; + cksum_type_t ret = OBD_CKSUM_ADLER; + + CDEBUG(D_INFO, "Crypto hash speed: crc %d, crc32c %d, adler %d\n", + cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_CRC32)), + cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_CRC32C)), + cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_ADLER))); + + if (cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_CRC32C)) > 0) + ret |= OBD_CKSUM_CRC32C; + if (cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_CRC32)) > 0) + ret |= OBD_CKSUM_CRC32; -#ifdef X86_FEATURE_XMM4_2 - if (cpu_has_xmm4_2) - ret |= OBD_CKSUM_CRC32C; -#endif -#ifdef HAVE_ADLER - ret |= OBD_CKSUM_ADLER; -#endif - return ret; + return ret; } +/* Server uses algos that perform at 50% or better of the Adler */ +static inline cksum_type_t cksum_types_supported_server(void) +{ + int base_speed; + cksum_type_t ret = OBD_CKSUM_ADLER; + + CDEBUG(D_INFO, "Crypto hash speed: crc %d, crc32c %d, adler %d\n", + cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_CRC32)), + cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_CRC32C)), + cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_ADLER))); + + base_speed = cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_ADLER)) / 2; + + if (cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_CRC32C)) >= + base_speed) + ret |= OBD_CKSUM_CRC32C; + if (cfs_crypto_hash_speed(cksum_obd2cfs(OBD_CKSUM_CRC32)) >= + base_speed) + ret |= OBD_CKSUM_CRC32; + + return ret; +} + + /* Select the best checksum algorithm among those supplied in the cksum_types * input. * * Currently, calling cksum_type_pack() with a mask will return the fastest - * checksum type due to its ordering, but in the future we might want to - * determine this based on benchmarking the different algorithms quickly. + * checksum type due to its benchmarking at libcfs module load. * Caution is advised, however, since what is fastest on a single client may * not be the fastest or most efficient algorithm on the server. */ static inline cksum_type_t cksum_type_select(cksum_type_t cksum_types) { - return cksum_type_unpack(cksum_type_pack(cksum_types)); + return cksum_type_unpack(cksum_type_pack(cksum_types)); } /* Checksum algorithm names. Must be defined in the same order as the