3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 only,
7 * as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License version 2 for more details (a copy is included
13 * in the LICENSE file that accompanied this code).
15 * You should have received a copy of the GNU General Public License
16 * version 2 along with this program; If not, see http://www.gnu.org/licenses
18 * Please visit http://www.xyratex.com/contact if you need additional
19 * information or have any questions.
25 * Copyright 2012 Xyratex Technology Limited
27 * Copyright (c) 2012, 2014, Intel Corporation.
30 #include <crypto/hash.h>
31 #include <linux/scatterlist.h>
32 #include <libcfs/libcfs.h>
33 #include <libcfs/libcfs_crypto.h>
34 #include <libcfs/linux/linux-crypto.h>
36 #ifndef HAVE_CRYPTO_HASH_HELPERS
37 static inline const char *crypto_ahash_alg_name(struct crypto_ahash *tfm)
39 return crypto_tfm_alg_name(crypto_ahash_tfm(tfm));
42 static inline const char *crypto_ahash_driver_name(struct crypto_ahash *tfm)
44 return crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm));
49 * Array of hash algorithm speed in MByte per second
51 static int cfs_crypto_hash_speeds[CFS_HASH_ALG_MAX];
54 * Initialize the state descriptor for the specified hash algorithm.
56 * An internal routine to allocate the hash-specific state in \a hdesc for
57 * use with cfs_crypto_hash_digest() to compute the hash of a single message,
58 * though possibly in multiple chunks. The descriptor internal state should
59 * be freed with cfs_crypto_hash_final().
61 * \param[in] hash_alg hash algorithm id (CFS_HASH_ALG_*)
62 * \param[out] type pointer to the hash description in hash_types[] array
63 * \param[in,out] req ahash request to be initialized
64 * \param[in] key initial hash value/state, NULL to use default value
65 * \param[in] key_len length of \a key
67 * \retval 0 on success
68 * \retval negative errno on failure
70 static int cfs_crypto_hash_alloc(enum cfs_crypto_hash_alg hash_alg,
71 const struct cfs_crypto_hash_type **type,
72 struct ahash_request **req,
76 struct crypto_ahash *tfm;
79 *type = cfs_crypto_hash_type(hash_alg);
82 CWARN("Unsupported hash algorithm id = %d, max id is %d\n",
83 hash_alg, CFS_HASH_ALG_MAX);
86 tfm = crypto_alloc_ahash((*type)->cht_name, 0, CRYPTO_ALG_ASYNC);
88 CDEBUG(D_INFO, "Failed to alloc crypto hash %s\n",
93 *req = ahash_request_alloc(tfm, GFP_KERNEL);
95 CDEBUG(D_INFO, "Failed to alloc ahash_request for %s\n",
97 crypto_free_ahash(tfm);
101 ahash_request_set_callback(*req, 0, NULL, NULL);
104 err = crypto_ahash_setkey(tfm, key, key_len);
105 else if ((*type)->cht_key != 0)
106 err = crypto_ahash_setkey(tfm,
107 (unsigned char *)&((*type)->cht_key),
111 ahash_request_free(*req);
112 crypto_free_ahash(tfm);
116 CDEBUG(D_INFO, "Using crypto hash: %s (%s) speed %d MB/s\n",
117 crypto_ahash_alg_name(tfm), crypto_ahash_driver_name(tfm),
118 cfs_crypto_hash_speeds[hash_alg]);
120 err = crypto_ahash_init(*req);
122 ahash_request_free(*req);
123 crypto_free_ahash(tfm);
129 * Calculate hash digest for the passed buffer.
131 * This should be used when computing the hash on a single contiguous buffer.
132 * It combines the hash initialization, computation, and cleanup.
134 * \param[in] hash_alg id of hash algorithm (CFS_HASH_ALG_*)
135 * \param[in] buf data buffer on which to compute hash
136 * \param[in] buf_len length of \a buf in bytes
137 * \param[in] key initial value/state for algorithm, if \a key = NULL
138 * use default initial value
139 * \param[in] key_len length of \a key in bytes
140 * \param[out] hash pointer to computed hash value, if \a hash = NULL then
141 * \a hash_len is to digest size in bytes, retval -ENOSPC
142 * \param[in,out] hash_len size of \a hash buffer
144 * \retval -EINVAL \a buf, \a buf_len, \a hash_len, \a hash_alg invalid
145 * \retval -ENOENT \a hash_alg is unsupported
146 * \retval -ENOSPC \a hash is NULL, or \a hash_len less than digest size
147 * \retval 0 for success
148 * \retval negative errno for other errors from lower layers.
150 int cfs_crypto_hash_digest(enum cfs_crypto_hash_alg hash_alg,
151 const void *buf, unsigned int buf_len,
152 unsigned char *key, unsigned int key_len,
153 unsigned char *hash, unsigned int *hash_len)
155 struct scatterlist sl;
156 struct ahash_request *req;
158 const struct cfs_crypto_hash_type *type;
160 if (!buf || buf_len == 0 || !hash_len)
163 err = cfs_crypto_hash_alloc(hash_alg, &type, &req, key, key_len);
167 if (!hash || *hash_len < type->cht_size) {
168 *hash_len = type->cht_size;
169 crypto_free_ahash(crypto_ahash_reqtfm(req));
170 ahash_request_free(req);
173 sg_init_one(&sl, (void *)buf, buf_len);
175 ahash_request_set_crypt(req, &sl, hash, sl.length);
176 err = crypto_ahash_digest(req);
177 crypto_free_ahash(crypto_ahash_reqtfm(req));
178 ahash_request_free(req);
182 EXPORT_SYMBOL(cfs_crypto_hash_digest);
185 * Allocate and initialize desriptor for hash algorithm.
187 * This should be used to initialize a hash descriptor for multiple calls
188 * to a single hash function when computing the hash across multiple
189 * separate buffers or pages using cfs_crypto_hash_update{,_page}().
191 * The hash descriptor should be freed with cfs_crypto_hash_final().
193 * \param[in] hash_alg algorithm id (CFS_HASH_ALG_*)
194 * \param[in] key initial value/state for algorithm, if \a key = NULL
195 * use default initial value
196 * \param[in] key_len length of \a key in bytes
198 * \retval pointer to descriptor of hash instance
199 * \retval ERR_PTR(errno) in case of error
201 struct cfs_crypto_hash_desc *
202 cfs_crypto_hash_init(enum cfs_crypto_hash_alg hash_alg,
203 unsigned char *key, unsigned int key_len)
205 struct ahash_request *req;
207 const struct cfs_crypto_hash_type *type;
209 err = cfs_crypto_hash_alloc(hash_alg, &type, &req, key, key_len);
212 return (struct cfs_crypto_hash_desc *)req;
214 EXPORT_SYMBOL(cfs_crypto_hash_init);
217 * Update hash digest computed on data within the given \a page
219 * \param[in] hdesc hash state descriptor
220 * \param[in] page data page on which to compute the hash
221 * \param[in] offset offset within \a page at which to start hash
222 * \param[in] len length of data on which to compute hash
224 * \retval 0 for success
225 * \retval negative errno on failure
227 int cfs_crypto_hash_update_page(struct cfs_crypto_hash_desc *hdesc,
228 struct page *page, unsigned int offset,
231 struct ahash_request *req = (void *)hdesc;
232 struct scatterlist sl;
234 sg_init_table(&sl, 1);
235 sg_set_page(&sl, page, len, offset & ~PAGE_MASK);
237 ahash_request_set_crypt(req, &sl, NULL, sl.length);
238 return crypto_ahash_update(req);
240 EXPORT_SYMBOL(cfs_crypto_hash_update_page);
243 * Update hash digest computed on the specified data
245 * \param[in] hdesc hash state descriptor
246 * \param[in] buf data buffer on which to compute the hash
247 * \param[in] buf_len length of \buf on which to compute hash
249 * \retval 0 for success
250 * \retval negative errno on failure
252 int cfs_crypto_hash_update(struct cfs_crypto_hash_desc *hdesc,
253 const void *buf, unsigned int buf_len)
255 struct ahash_request *req = (void *)hdesc;
256 struct scatterlist sl;
258 sg_init_one(&sl, (void *)buf, buf_len);
260 ahash_request_set_crypt(req, &sl, NULL, sl.length);
261 return crypto_ahash_update(req);
263 EXPORT_SYMBOL(cfs_crypto_hash_update);
266 * Finish hash calculation, copy hash digest to buffer, clean up hash descriptor
268 * \param[in] hdesc hash descriptor
269 * \param[out] hash pointer to hash buffer to store hash digest
270 * \param[in,out] hash_len pointer to hash buffer size, if \a hash == NULL
271 * or hash_len == NULL only free \a hdesc instead
272 * of computing the hash
274 * \retval 0 for success
275 * \retval -EOVERFLOW if hash_len is too small for the hash digest
276 * \retval negative errno for other errors from lower layers
278 int cfs_crypto_hash_final(struct cfs_crypto_hash_desc *hdesc,
279 unsigned char *hash, unsigned int *hash_len)
281 struct ahash_request *req = (void *)hdesc;
282 int size = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
285 if (!hash || !hash_len) {
289 if (*hash_len < size) {
294 ahash_request_set_crypt(req, NULL, hash, 0);
295 err = crypto_ahash_final(req);
299 crypto_free_ahash(crypto_ahash_reqtfm(req));
300 ahash_request_free(req);
304 EXPORT_SYMBOL(cfs_crypto_hash_final);
307 * Compute the speed of specified hash function
309 * Run a speed test on the given hash algorithm on buffer of the given size.
310 * The speed is stored internally in the cfs_crypto_hash_speeds[] array, and
311 * is available through the cfs_crypto_hash_speed() function.
313 * \param[in] hash_alg hash algorithm id (CFS_HASH_ALG_*)
314 * \param[in] buf data buffer on which to compute the hash
315 * \param[in] buf_len length of \buf on which to compute hash
317 static void cfs_crypto_performance_test(enum cfs_crypto_hash_alg hash_alg)
319 int buf_len = max(PAGE_SIZE, 1048576UL);
321 unsigned long start, end;
324 unsigned char hash[CFS_CRYPTO_HASH_DIGESTSIZE_MAX];
325 unsigned int hash_len = sizeof(hash);
327 page = alloc_page(GFP_KERNEL);
334 memset(buf, 0xAD, PAGE_SIZE);
337 for (start = jiffies, end = start + msecs_to_jiffies(MSEC_PER_SEC),
339 time_before(jiffies, end) && err == 0; bcount++) {
340 struct cfs_crypto_hash_desc *hdesc;
343 hdesc = cfs_crypto_hash_init(hash_alg, NULL, 0);
345 err = PTR_ERR(hdesc);
349 for (i = 0; i < buf_len / PAGE_SIZE; i++) {
350 err = cfs_crypto_hash_update_page(hdesc, page, 0,
356 err = cfs_crypto_hash_final(hdesc, hash, &hash_len);
364 cfs_crypto_hash_speeds[hash_alg] = err;
365 CDEBUG(D_INFO, "Crypto hash algorithm %s test error: rc = %d\n",
366 cfs_crypto_hash_name(hash_alg), err);
370 tmp = ((bcount * buf_len / jiffies_to_msecs(end - start)) *
371 1000) / (1024 * 1024);
372 cfs_crypto_hash_speeds[hash_alg] = (int)tmp;
373 CDEBUG(D_CONFIG, "Crypto hash algorithm %s speed = %d MB/s\n",
374 cfs_crypto_hash_name(hash_alg),
375 cfs_crypto_hash_speeds[hash_alg]);
380 * hash speed in Mbytes per second for valid hash algorithm
382 * Return the performance of the specified \a hash_alg that was previously
383 * computed using cfs_crypto_performance_test().
385 * \param[in] hash_alg hash algorithm id (CFS_HASH_ALG_*)
387 * \retval positive speed of the hash function in MB/s
388 * \retval -ENOENT if \a hash_alg is unsupported
389 * \retval negative errno if \a hash_alg speed is unavailable
391 int cfs_crypto_hash_speed(enum cfs_crypto_hash_alg hash_alg)
393 if (hash_alg < CFS_HASH_ALG_MAX)
394 return cfs_crypto_hash_speeds[hash_alg];
398 EXPORT_SYMBOL(cfs_crypto_hash_speed);
401 * Run the performance test for all hash algorithms.
403 * Run the cfs_crypto_performance_test() benchmark for all of the available
404 * hash functions using a 1MB buffer size. This is a reasonable buffer size
405 * for Lustre RPCs, even if the actual RPC size is larger or smaller.
407 * Since the setup cost and computation speed of various hash algorithms is
408 * a function of the buffer size (and possibly internal contention of offload
409 * engines), this speed only represents an estimate of the actual speed under
410 * actual usage, but is reasonable for comparing available algorithms.
412 * The actual speeds are available via cfs_crypto_hash_speed() for later
415 * \retval 0 on success
416 * \retval -ENOMEM if no memory is available for test buffer
418 static int cfs_crypto_test_hashes(void)
420 enum cfs_crypto_hash_alg hash_alg;
422 for (hash_alg = 0; hash_alg < CFS_HASH_ALG_MAX; hash_alg++)
423 cfs_crypto_performance_test(hash_alg);
433 #ifdef HAVE_PCLMULQDQ
434 #ifdef NEED_CRC32_ACCEL
435 static int crc32_pclmul;
437 #ifdef NEED_CRC32C_ACCEL
438 static int crc32c_pclmul;
440 #endif /* HAVE_PCLMULQDQ */
443 * Register available hash functions
447 int cfs_crypto_register(void)
449 request_module("crc32c");
451 adler32 = cfs_crypto_adler32_register();
454 crc32 = cfs_crypto_crc32_register();
456 #ifdef HAVE_PCLMULQDQ
457 #ifdef NEED_CRC32_ACCEL
458 crc32_pclmul = cfs_crypto_crc32_pclmul_register();
460 #ifdef NEED_CRC32C_ACCEL
461 crc32c_pclmul = cfs_crypto_crc32c_pclmul_register();
463 #endif /* HAVE_PCLMULQDQ */
465 /* check all algorithms and do performance test */
466 cfs_crypto_test_hashes();
472 * Unregister previously registered hash functions
474 void cfs_crypto_unregister(void)
477 cfs_crypto_adler32_unregister();
481 cfs_crypto_crc32_unregister();
483 #ifdef HAVE_PCLMULQDQ
484 #ifdef NEED_CRC32_ACCEL
485 if (crc32_pclmul == 0)
486 cfs_crypto_crc32_pclmul_unregister();
488 #ifdef NEED_CRC32C_ACCEL
489 if (crc32c_pclmul == 0)
490 cfs_crypto_crc32c_pclmul_unregister();
492 #endif /* HAVE_PCLMULQDQ */