X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=libcfs%2Flibcfs%2Flinux%2Flinux-crypto.c;h=1a0a1055ac34a9d8ced462f6651580457d9c5519;hp=a15938ac68f93077242633a97aaffd7c52bb2917;hb=0c8d53e17be600c99e4a8f96062f39306c3ccad8;hpb=84a3fd67356c8073a917ea6abd63928055e38156 diff --git a/libcfs/libcfs/linux/linux-crypto.c b/libcfs/libcfs/linux/linux-crypto.c index a15938a..1a0a105 100644 --- a/libcfs/libcfs/linux/linux-crypto.c +++ b/libcfs/libcfs/linux/linux-crypto.c @@ -23,336 +23,472 @@ /* * Copyright 2012 Xyratex Technology Limited + * + * Copyright (c) 2012, 2014, Intel Corporation. */ -#include +#include #include #include +#include #include -/** - * Array of hash algorithm speed in MByte per second - */ -static int cfs_crypto_hash_speeds[CFS_HASH_ALG_MAX]; - - -#ifndef HAVE_STRUCT_HASH_DESC -/** 2.6.18 kernel have no crypto_hash function - * this part was copied from lustre_compat25.h */ -#define crypto_hash crypto_tfm -struct hash_desc { - struct crypto_hash *tfm; - unsigned int flags; -}; - -static inline -struct crypto_hash *crypto_alloc_hash(const char *alg, unsigned int type, - unsigned int mask) -{ - return crypto_alloc_tfm(alg, 0); -} -static inline void crypto_free_hash(struct crypto_hash *tfm) +#ifndef HAVE_CRYPTO_HASH_HELPERS +static inline const char *crypto_ahash_alg_name(struct crypto_ahash *tfm) { - crypto_free_tfm(tfm); + return crypto_tfm_alg_name(crypto_ahash_tfm(tfm)); } -static inline int crypto_hash_init(struct hash_desc *desc) +static inline const char *crypto_ahash_driver_name(struct crypto_ahash *tfm) { - crypto_digest_init(desc->tfm); - return 0; + return crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm)); } - -static inline int crypto_hash_update(struct hash_desc *desc, - struct scatterlist *sg, - unsigned int nbytes) -{ - if (desc->tfm->crt_digest.dit_update == NULL) - return -1; - - LASSERT(nbytes == sg->length); - crypto_digest_update(desc->tfm, sg, 1); - - return 0; -} - -static inline int crypto_hash_digest(struct hash_desc *desc, - struct scatterlist *sg, - unsigned int nbytes, unsigned char *out) -{ - crypto_hash_update(desc, sg, nbytes); - crypto_digest_final(desc->tfm, out); - return 0; -} - -static inline int crypto_hash_final(struct hash_desc *desc, unsigned char *out) -{ - crypto_digest_final(desc->tfm, out); - return 0; -} - -static inline struct crypto_tfm *crypto_hash_tfm(struct crypto_hash *tfm) -{ - return tfm; -} - -#define crypto_hash_setkey(tfm, key, keylen) \ - crypto_digest_setkey(tfm, key, keylen) -#define crypto_hash_digestsize(tfm) crypto_tfm_alg_digestsize(tfm) -#define crypto_hash_blocksize(tfm) crypto_tfm_alg_blocksize(tfm) #endif -static int cfs_crypto_hash_alloc(unsigned char alg_id, +/** + * Array of hash algorithm speed in MByte per second + */ +static int cfs_crypto_hash_speeds[CFS_HASH_ALG_MAX]; + +/** + * Initialize the state descriptor for the specified hash algorithm. + * + * An internal routine to allocate the hash-specific state in \a hdesc for + * use with cfs_crypto_hash_digest() to compute the hash of a single message, + * though possibly in multiple chunks. The descriptor internal state should + * be freed with cfs_crypto_hash_final(). + * + * \param[in] hash_alg hash algorithm id (CFS_HASH_ALG_*) + * \param[out] type pointer to the hash description in hash_types[] array + * \param[in,out] req ahash request to be initialized + * \param[in] key initial hash value/state, NULL to use default value + * \param[in] key_len length of \a key + * + * \retval 0 on success + * \retval negative errno on failure + */ +static int cfs_crypto_hash_alloc(enum cfs_crypto_hash_alg hash_alg, const struct cfs_crypto_hash_type **type, - struct hash_desc *desc, unsigned char *key, + struct ahash_request **req, + unsigned char *key, unsigned int key_len) { - int err = 0; + struct crypto_ahash *tfm; + int err = 0; - *type = cfs_crypto_hash_type(alg_id); + *type = cfs_crypto_hash_type(hash_alg); if (*type == NULL) { CWARN("Unsupported hash algorithm id = %d, max id is %d\n", - alg_id, CFS_HASH_ALG_MAX); + hash_alg, CFS_HASH_ALG_MAX); return -EINVAL; } - desc->tfm = crypto_alloc_hash((*type)->cht_name, 0, 0); - - if (desc->tfm == NULL) - return -EINVAL; - - if (IS_ERR(desc->tfm)) { + tfm = crypto_alloc_ahash((*type)->cht_name, 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(tfm)) { CDEBUG(D_INFO, "Failed to alloc crypto hash %s\n", (*type)->cht_name); - return PTR_ERR(desc->tfm); + return PTR_ERR(tfm); } - desc->flags = 0; + *req = ahash_request_alloc(tfm, GFP_KERNEL); + if (!*req) { + CDEBUG(D_INFO, "Failed to alloc ahash_request for %s\n", + (*type)->cht_name); + crypto_free_ahash(tfm); + return -ENOMEM; + } - /** Shash have different logic for initialization then digest - * shash: crypto_hash_setkey, crypto_hash_init - * digest: crypto_digest_init, crypto_digest_setkey - * Skip this function for digest, because we use shash logic at - * cfs_crypto_hash_alloc. - */ -#ifndef HAVE_STRUCT_SHASH_ALG - crypto_hash_init(desc); -#endif - if (key != NULL) { - err = crypto_hash_setkey(desc->tfm, key, key_len); - } else if ((*type)->cht_key != 0) { - err = crypto_hash_setkey(desc->tfm, + ahash_request_set_callback(*req, 0, NULL, NULL); + + if (key) + err = crypto_ahash_setkey(tfm, key, key_len); + else if ((*type)->cht_key != 0) + err = crypto_ahash_setkey(tfm, (unsigned char *)&((*type)->cht_key), (*type)->cht_size); - } if (err != 0) { - crypto_free_hash(desc->tfm); + ahash_request_free(*req); + crypto_free_ahash(tfm); return err; } CDEBUG(D_INFO, "Using crypto hash: %s (%s) speed %d MB/s\n", - crypto_tfm_alg_name(crypto_hash_tfm(desc->tfm)), - crypto_tfm_alg_driver_name(crypto_hash_tfm(desc->tfm)), - cfs_crypto_hash_speeds[alg_id]); + crypto_ahash_alg_name(tfm), crypto_ahash_driver_name(tfm), + cfs_crypto_hash_speeds[hash_alg]); -#ifdef HAVE_STRUCT_SHASH_ALG - return crypto_hash_init(desc); -#else - return 0; -#endif + err = crypto_ahash_init(*req); + if (err) { + ahash_request_free(*req); + crypto_free_ahash(tfm); + } + return err; } -int cfs_crypto_hash_digest(unsigned char alg_id, +/** + * Calculate hash digest for the passed buffer. + * + * This should be used when computing the hash on a single contiguous buffer. + * It combines the hash initialization, computation, and cleanup. + * + * \param[in] hash_alg id of hash algorithm (CFS_HASH_ALG_*) + * \param[in] buf data buffer on which to compute hash + * \param[in] buf_len length of \a buf in bytes + * \param[in] key initial value/state for algorithm, if \a key = NULL + * use default initial value + * \param[in] key_len length of \a key in bytes + * \param[out] hash pointer to computed hash value, if \a hash = NULL then + * \a hash_len is to digest size in bytes, retval -ENOSPC + * \param[in,out] hash_len size of \a hash buffer + * + * \retval -EINVAL \a buf, \a buf_len, \a hash_len, \a hash_alg invalid + * \retval -ENOENT \a hash_alg is unsupported + * \retval -ENOSPC \a hash is NULL, or \a hash_len less than digest size + * \retval 0 for success + * \retval negative errno for other errors from lower layers. + */ +int cfs_crypto_hash_digest(enum cfs_crypto_hash_alg hash_alg, const void *buf, unsigned int buf_len, unsigned char *key, unsigned int key_len, unsigned char *hash, unsigned int *hash_len) { - struct scatterlist sl = {0}; - struct hash_desc hdesc; + struct scatterlist sl; + struct ahash_request *req; int err; const struct cfs_crypto_hash_type *type; - if (buf == NULL || buf_len == 0 || hash_len == NULL) + if (!buf || buf_len == 0 || !hash_len) return -EINVAL; - err = cfs_crypto_hash_alloc(alg_id, &type, &hdesc, key, key_len); + err = cfs_crypto_hash_alloc(hash_alg, &type, &req, key, key_len); if (err != 0) return err; - if (hash == NULL || *hash_len < type->cht_size) { + if (!hash || *hash_len < type->cht_size) { *hash_len = type->cht_size; - crypto_free_hash(hdesc.tfm); + crypto_free_ahash(crypto_ahash_reqtfm(req)); + ahash_request_free(req); return -ENOSPC; } - sg_set_buf(&sl, (void *)buf, buf_len); + sg_init_one(&sl, (void *)buf, buf_len); - hdesc.flags = 0; - err = crypto_hash_digest(&hdesc, &sl, sl.length, hash); - crypto_free_hash(hdesc.tfm); + ahash_request_set_crypt(req, &sl, hash, sl.length); + err = crypto_ahash_digest(req); + crypto_free_ahash(crypto_ahash_reqtfm(req)); + ahash_request_free(req); return err; } EXPORT_SYMBOL(cfs_crypto_hash_digest); +/** + * Allocate and initialize desriptor for hash algorithm. + * + * This should be used to initialize a hash descriptor for multiple calls + * to a single hash function when computing the hash across multiple + * separate buffers or pages using cfs_crypto_hash_update{,_page}(). + * + * The hash descriptor should be freed with cfs_crypto_hash_final(). + * + * \param[in] hash_alg algorithm id (CFS_HASH_ALG_*) + * \param[in] key initial value/state for algorithm, if \a key = NULL + * use default initial value + * \param[in] key_len length of \a key in bytes + * + * \retval pointer to descriptor of hash instance + * \retval ERR_PTR(errno) in case of error + */ struct cfs_crypto_hash_desc * - cfs_crypto_hash_init(unsigned char alg_id, + cfs_crypto_hash_init(enum cfs_crypto_hash_alg hash_alg, unsigned char *key, unsigned int key_len) { - - struct hash_desc *hdesc; - int err; + struct ahash_request *req; + int err; const struct cfs_crypto_hash_type *type; - hdesc = cfs_alloc(sizeof(*hdesc), 0); - if (hdesc == NULL) - return ERR_PTR(-ENOMEM); - - err = cfs_crypto_hash_alloc(alg_id, &type, hdesc, key, key_len); - - if (err) { - cfs_free(hdesc); + err = cfs_crypto_hash_alloc(hash_alg, &type, &req, key, key_len); + if (err) return ERR_PTR(err); - } - return (struct cfs_crypto_hash_desc *)hdesc; + return (struct cfs_crypto_hash_desc *)req; } EXPORT_SYMBOL(cfs_crypto_hash_init); +/** + * Update hash digest computed on data within the given \a page + * + * \param[in] hdesc hash state descriptor + * \param[in] page data page on which to compute the hash + * \param[in] offset offset within \a page at which to start hash + * \param[in] len length of data on which to compute hash + * + * \retval 0 for success + * \retval negative errno on failure + */ int cfs_crypto_hash_update_page(struct cfs_crypto_hash_desc *hdesc, - cfs_page_t *page, unsigned int offset, + struct page *page, unsigned int offset, unsigned int len) { - struct scatterlist sl = {0}; + struct ahash_request *req = (void *)hdesc; + struct scatterlist sl; - sg_set_page(&sl, page, len, offset & ~CFS_PAGE_MASK); + sg_init_table(&sl, 1); + sg_set_page(&sl, page, len, offset & ~PAGE_MASK); - return crypto_hash_update((struct hash_desc *)hdesc, &sl, sl.length); + ahash_request_set_crypt(req, &sl, NULL, sl.length); + return crypto_ahash_update(req); } EXPORT_SYMBOL(cfs_crypto_hash_update_page); +/** + * Update hash digest computed on the specified data + * + * \param[in] hdesc hash state descriptor + * \param[in] buf data buffer on which to compute the hash + * \param[in] buf_len length of \buf on which to compute hash + * + * \retval 0 for success + * \retval negative errno on failure + */ int cfs_crypto_hash_update(struct cfs_crypto_hash_desc *hdesc, const void *buf, unsigned int buf_len) { - struct scatterlist sl = {0}; + struct ahash_request *req = (void *)hdesc; + struct scatterlist sl; - sg_set_buf(&sl, (void *)buf, buf_len); + sg_init_one(&sl, (void *)buf, buf_len); - return crypto_hash_update((struct hash_desc *)hdesc, &sl, sl.length); + ahash_request_set_crypt(req, &sl, NULL, sl.length); + return crypto_ahash_update(req); } EXPORT_SYMBOL(cfs_crypto_hash_update); -/* If hash_len pointer is NULL - destroy descriptor. */ +/** + * Finish hash calculation, copy hash digest to buffer, clean up hash descriptor + * + * \param[in] hdesc hash descriptor + * \param[out] hash pointer to hash buffer to store hash digest + * \param[in,out] hash_len pointer to hash buffer size, if \a hash == NULL + * or hash_len == NULL only free \a hdesc instead + * of computing the hash + * + * \retval 0 for success + * \retval -EOVERFLOW if hash_len is too small for the hash digest + * \retval negative errno for other errors from lower layers + */ int cfs_crypto_hash_final(struct cfs_crypto_hash_desc *hdesc, unsigned char *hash, unsigned int *hash_len) { - int err; - int size = crypto_hash_digestsize(((struct hash_desc *)hdesc)->tfm); + struct ahash_request *req = (void *)hdesc; + int size = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); + int err; - if (hash_len == NULL) { - crypto_free_hash(((struct hash_desc *)hdesc)->tfm); - return 0; + if (!hash || !hash_len) { + err = 0; + goto free; } - if (hash == NULL || *hash_len < size) { - *hash_len = size; - return -ENOSPC; + if (*hash_len < size) { + err = -EOVERFLOW; + goto free; } - err = crypto_hash_final((struct hash_desc *) hdesc, hash); - if (err < 0) { - /* May be caller can fix error */ - return err; - } - crypto_free_hash(((struct hash_desc *)hdesc)->tfm); + ahash_request_set_crypt(req, NULL, hash, 0); + err = crypto_ahash_final(req); + if (err == 0) + *hash_len = size; +free: + crypto_free_ahash(crypto_ahash_reqtfm(req)); + ahash_request_free(req); + return err; } EXPORT_SYMBOL(cfs_crypto_hash_final); -static void cfs_crypto_performance_test(unsigned char alg_id, - const unsigned char *buf, - unsigned int buf_len) +/** + * Compute the speed of specified hash function + * + * Run a speed test on the given hash algorithm on buffer of the given size. + * The speed is stored internally in the cfs_crypto_hash_speeds[] array, and + * is available through the cfs_crypto_hash_speed() function. + * + * \param[in] hash_alg hash algorithm id (CFS_HASH_ALG_*) + * \param[in] buf data buffer on which to compute the hash + * \param[in] buf_len length of \buf on which to compute hash + */ +static void cfs_crypto_performance_test(enum cfs_crypto_hash_alg hash_alg) { - unsigned long start, end; - int bcount, err = 0; - int sec = 1; /* do test only 1 sec */ - unsigned char hash[64]; - unsigned int hash_len = 64; - - for (start = jiffies, end = start + sec * HZ, bcount = 0; - time_before(jiffies, end); bcount++) { - err = cfs_crypto_hash_digest(alg_id, buf, buf_len, NULL, 0, - hash, &hash_len); - if (err) + int buf_len = max(PAGE_SIZE, 1048576UL); + void *buf; + unsigned long start, end; + int err = 0; + unsigned long bcount; + struct page *page; + unsigned char hash[CFS_CRYPTO_HASH_DIGESTSIZE_MAX]; + unsigned int hash_len = sizeof(hash); + + page = alloc_page(GFP_KERNEL); + if (page == NULL) { + err = -ENOMEM; + goto out_err; + } + + buf = kmap(page); + memset(buf, 0xAD, PAGE_SIZE); + kunmap(page); + + for (start = jiffies, end = start + msecs_to_jiffies(MSEC_PER_SEC), + bcount = 0; + time_before(jiffies, end) && err == 0; bcount++) { + struct cfs_crypto_hash_desc *hdesc; + int i; + + hdesc = cfs_crypto_hash_init(hash_alg, NULL, 0); + if (IS_ERR(hdesc)) { + err = PTR_ERR(hdesc); break; + } + + for (i = 0; i < buf_len / PAGE_SIZE; i++) { + err = cfs_crypto_hash_update_page(hdesc, page, 0, + PAGE_SIZE); + if (err != 0) + break; + } + err = cfs_crypto_hash_final(hdesc, hash, &hash_len); + if (err != 0) + break; } end = jiffies; - - if (err) { - cfs_crypto_hash_speeds[alg_id] = -1; - CDEBUG(D_INFO, "Crypto hash algorithm %s, err = %d\n", - cfs_crypto_hash_name(alg_id), err); + __free_page(page); +out_err: + if (err != 0) { + cfs_crypto_hash_speeds[hash_alg] = err; + CDEBUG(D_INFO, "Crypto hash algorithm %s test error: rc = %d\n", + cfs_crypto_hash_name(hash_alg), err); } else { unsigned long tmp; + tmp = ((bcount * buf_len / jiffies_to_msecs(end - start)) * 1000) / (1024 * 1024); - cfs_crypto_hash_speeds[alg_id] = (int)tmp; + cfs_crypto_hash_speeds[hash_alg] = (int)tmp; + CDEBUG(D_CONFIG, "Crypto hash algorithm %s speed = %d MB/s\n", + cfs_crypto_hash_name(hash_alg), + cfs_crypto_hash_speeds[hash_alg]); } - CDEBUG(D_INFO, "Crypto hash algorithm %s speed = %d MB/s\n", - cfs_crypto_hash_name(alg_id), cfs_crypto_hash_speeds[alg_id]); } -int cfs_crypto_hash_speed(unsigned char hash_alg) +/** + * hash speed in Mbytes per second for valid hash algorithm + * + * Return the performance of the specified \a hash_alg that was previously + * computed using cfs_crypto_performance_test(). + * + * \param[in] hash_alg hash algorithm id (CFS_HASH_ALG_*) + * + * \retval positive speed of the hash function in MB/s + * \retval -ENOENT if \a hash_alg is unsupported + * \retval negative errno if \a hash_alg speed is unavailable + */ +int cfs_crypto_hash_speed(enum cfs_crypto_hash_alg hash_alg) { if (hash_alg < CFS_HASH_ALG_MAX) return cfs_crypto_hash_speeds[hash_alg]; - else - return -1; + + return -ENOENT; } EXPORT_SYMBOL(cfs_crypto_hash_speed); /** - * Do performance test for all hash algorithms. + * Run the performance test for all hash algorithms. + * + * Run the cfs_crypto_performance_test() benchmark for all of the available + * hash functions using a 1MB buffer size. This is a reasonable buffer size + * for Lustre RPCs, even if the actual RPC size is larger or smaller. + * + * Since the setup cost and computation speed of various hash algorithms is + * a function of the buffer size (and possibly internal contention of offload + * engines), this speed only represents an estimate of the actual speed under + * actual usage, but is reasonable for comparing available algorithms. + * + * The actual speeds are available via cfs_crypto_hash_speed() for later + * comparison. + * + * \retval 0 on success + * \retval -ENOMEM if no memory is available for test buffer */ static int cfs_crypto_test_hashes(void) { - unsigned char i; - unsigned char *data; - unsigned int j; - /* Data block size for testing hash. Maximum - * kmalloc size for 2.6.18 kernel is 128K */ - unsigned int data_len = 1 * 128 * 1024; - - data = cfs_alloc(data_len, 0); - if (data == NULL) { - CERROR("Failed to allocate mem\n"); - return -ENOMEM; - } + enum cfs_crypto_hash_alg hash_alg; - for (j = 0; j < data_len; j++) - data[j] = j & 0xff; + for (hash_alg = 0; hash_alg < CFS_HASH_ALG_MAX; hash_alg++) + cfs_crypto_performance_test(hash_alg); - for (i = 0; i < CFS_HASH_ALG_MAX; i++) - cfs_crypto_performance_test(i, data, data_len); - - cfs_free(data); return 0; } -static int crc32, adler32; +static int adler32; +#ifdef HAVE_CRC32 +static int crc32; +#endif +#ifdef HAVE_PCLMULQDQ +#ifdef NEED_CRC32_ACCEL +static int crc32_pclmul; +#endif +#ifdef NEED_CRC32C_ACCEL +static int crc32c_pclmul; +#endif +#endif /* HAVE_PCLMULQDQ */ + +/** + * Register available hash functions + * + * \retval 0 + */ int cfs_crypto_register(void) { - crc32 = cfs_crypto_crc32_register(); + request_module("crc32c"); + adler32 = cfs_crypto_adler32_register(); - /* check all algorithms and do perfermance test */ +#ifdef HAVE_CRC32 + crc32 = cfs_crypto_crc32_register(); +#endif +#ifdef HAVE_PCLMULQDQ +#ifdef NEED_CRC32_ACCEL + crc32_pclmul = cfs_crypto_crc32_pclmul_register(); +#endif +#ifdef NEED_CRC32C_ACCEL + crc32c_pclmul = cfs_crypto_crc32c_pclmul_register(); +#endif +#endif /* HAVE_PCLMULQDQ */ + + /* check all algorithms and do performance test */ cfs_crypto_test_hashes(); + return 0; } + +/** + * Unregister previously registered hash functions + */ void cfs_crypto_unregister(void) { - if (crc32 == 0) - cfs_crypto_crc32_unregister(); if (adler32 == 0) cfs_crypto_adler32_unregister(); - return; + +#ifdef HAVE_CRC32 + if (crc32 == 0) + cfs_crypto_crc32_unregister(); +#endif +#ifdef HAVE_PCLMULQDQ +#ifdef NEED_CRC32_ACCEL + if (crc32_pclmul == 0) + cfs_crypto_crc32_pclmul_unregister(); +#endif +#ifdef NEED_CRC32C_ACCEL + if (crc32c_pclmul == 0) + cfs_crypto_crc32c_pclmul_unregister(); +#endif +#endif /* HAVE_PCLMULQDQ */ }