3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 only,
7 * as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License version 2 for more details (a copy is included
13 * in the LICENSE file that accompanied this code).
15 * You should have received a copy of the GNU General Public License
16 * version 2 along with this program; If not, see http://www.gnu.org/licenses
18 * Please visit http://www.xyratex.com/contact if you need additional
19 * information or have any questions.
25 * Copyright 2012 Xyratex Technology Limited
27 * Copyright (c) 2012, 2014, Intel Corporation.
30 #include <crypto/hash.h>
31 #include <linux/scatterlist.h>
32 #include <linux/pagemap.h>
33 #include <libcfs/libcfs.h>
34 #include <libcfs/libcfs_crypto.h>
35 #include "linux-crypto.h"
37 #ifndef HAVE_CRYPTO_HASH_HELPERS
38 static inline const char *crypto_ahash_alg_name(struct crypto_ahash *tfm)
40 return crypto_tfm_alg_name(crypto_ahash_tfm(tfm));
43 static inline const char *crypto_ahash_driver_name(struct crypto_ahash *tfm)
45 return crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm));
50 * Array of hash algorithm speed in MByte per second
52 int cfs_crypto_hash_speeds[CFS_HASH_ALG_MAX];
53 EXPORT_SYMBOL(cfs_crypto_hash_speeds);
56 * Initialize the state descriptor for the specified hash algorithm.
58 * An internal routine to allocate the hash-specific state in \a hdesc for
59 * use with cfs_crypto_hash_digest() to compute the hash of a single message,
60 * though possibly in multiple chunks. The descriptor internal state should
61 * be freed with cfs_crypto_hash_final().
63 * \param[in] hash_alg hash algorithm id (CFS_HASH_ALG_*)
64 * \param[out] type pointer to the hash description in hash_types[] array
65 * \param[in,out] req ahash request to be initialized
66 * \param[in] key initial hash value/state, NULL to use default value
67 * \param[in] key_len length of \a key
69 * \retval 0 on success
70 * \retval negative errno on failure
72 static int cfs_crypto_hash_alloc(enum cfs_crypto_hash_alg hash_alg,
73 const struct cfs_crypto_hash_type **type,
74 struct ahash_request **req,
78 struct crypto_ahash *tfm;
81 *type = cfs_crypto_hash_type(hash_alg);
83 CWARN("Unsupported hash algorithm id = %d, max id is %d\n",
84 hash_alg, CFS_HASH_ALG_MAX);
88 /* Keys are only supported for the hmac version */
89 if (key && key_len > 0) {
92 algo_name = kasprintf(GFP_KERNEL, "hmac(%s)",
97 tfm = crypto_alloc_ahash(algo_name, 0, CRYPTO_ALG_ASYNC);
100 tfm = crypto_alloc_ahash((*type)->cht_name, 0,
104 CDEBUG_LIMIT(PTR_ERR(tfm) == -ENOMEM ? D_ERROR : D_INFO,
105 "Failed to alloc crypto hash %s: rc = %d\n",
106 (*type)->cht_name, (int)PTR_ERR(tfm));
110 *req = ahash_request_alloc(tfm, GFP_KERNEL);
112 CDEBUG(D_INFO, "Failed to alloc ahash_request for %s\n",
114 GOTO(out_free_tfm, err = -ENOMEM);
117 ahash_request_set_callback(*req, 0, NULL, NULL);
120 err = crypto_ahash_setkey(tfm, key, key_len);
121 else if ((*type)->cht_key != 0)
122 err = crypto_ahash_setkey(tfm,
123 (unsigned char *)&((*type)->cht_key),
126 GOTO(out_free_req, err);
128 CDEBUG(D_INFO, "Using crypto hash: %s (%s) speed %d MB/s\n",
129 crypto_ahash_alg_name(tfm), crypto_ahash_driver_name(tfm),
130 cfs_crypto_hash_speeds[hash_alg]);
132 err = crypto_ahash_init(*req);
135 ahash_request_free(*req);
137 crypto_free_ahash(tfm);
143 * Calculate hash digest for the passed buffer.
145 * This should be used when computing the hash on a single contiguous buffer.
146 * It combines the hash initialization, computation, and cleanup.
148 * \param[in] hash_alg id of hash algorithm (CFS_HASH_ALG_*)
149 * \param[in] buf data buffer on which to compute hash
150 * \param[in] buf_len length of \a buf in bytes
151 * \param[in] key initial value/state for algorithm, if \a key = NULL
152 * use default initial value
153 * \param[in] key_len length of \a key in bytes
154 * \param[out] hash pointer to computed hash value, if \a hash = NULL then
155 * \a hash_len is to digest size in bytes, retval -ENOSPC
156 * \param[in,out] hash_len size of \a hash buffer
158 * \retval -EINVAL \a buf, \a buf_len, \a hash_len, \a hash_alg invalid
159 * \retval -ENOENT \a hash_alg is unsupported
160 * \retval -ENOSPC \a hash is NULL, or \a hash_len less than digest size
161 * \retval 0 for success
162 * \retval negative errno for other errors from lower layers.
164 int cfs_crypto_hash_digest(enum cfs_crypto_hash_alg hash_alg,
165 const void *buf, unsigned int buf_len,
166 unsigned char *key, unsigned int key_len,
167 unsigned char *hash, unsigned int *hash_len)
169 struct scatterlist sl;
170 struct ahash_request *req;
172 const struct cfs_crypto_hash_type *type;
174 if (!buf || buf_len == 0 || !hash_len)
177 err = cfs_crypto_hash_alloc(hash_alg, &type, &req, key, key_len);
181 if (!hash || *hash_len < type->cht_size) {
182 *hash_len = type->cht_size;
183 crypto_free_ahash(crypto_ahash_reqtfm(req));
184 ahash_request_free(req);
187 sg_init_one(&sl, (void *)buf, buf_len);
189 ahash_request_set_crypt(req, &sl, hash, sl.length);
190 err = crypto_ahash_digest(req);
191 crypto_free_ahash(crypto_ahash_reqtfm(req));
192 ahash_request_free(req);
196 EXPORT_SYMBOL(cfs_crypto_hash_digest);
199 * Allocate and initialize desriptor for hash algorithm.
201 * This should be used to initialize a hash descriptor for multiple calls
202 * to a single hash function when computing the hash across multiple
203 * separate buffers or pages using cfs_crypto_hash_update{,_page}().
205 * The hash descriptor should be freed with cfs_crypto_hash_final().
207 * \param[in] hash_alg algorithm id (CFS_HASH_ALG_*)
208 * \param[in] key initial value/state for algorithm, if \a key = NULL
209 * use default initial value
210 * \param[in] key_len length of \a key in bytes
212 * \retval pointer to ahash request
213 * \retval ERR_PTR(errno) in case of error
215 struct ahash_request *
216 cfs_crypto_hash_init(enum cfs_crypto_hash_alg hash_alg,
217 unsigned char *key, unsigned int key_len)
219 struct ahash_request *req;
221 const struct cfs_crypto_hash_type *type;
223 err = cfs_crypto_hash_alloc(hash_alg, &type, &req, key, key_len);
228 EXPORT_SYMBOL(cfs_crypto_hash_init);
231 * Update hash digest computed on data within the given \a page
233 * \param[in] req ahash request
234 * \param[in] page data page on which to compute the hash
235 * \param[in] offset offset within \a page at which to start hash
236 * \param[in] len length of data on which to compute hash
238 * \retval 0 for success
239 * \retval negative errno on failure
241 int cfs_crypto_hash_update_page(struct ahash_request *req,
242 struct page *page, unsigned int offset,
245 struct scatterlist sl;
247 sg_init_table(&sl, 1);
248 sg_set_page(&sl, page, len, offset & ~PAGE_MASK);
250 ahash_request_set_crypt(req, &sl, NULL, sl.length);
251 return crypto_ahash_update(req);
253 EXPORT_SYMBOL(cfs_crypto_hash_update_page);
256 * Update hash digest computed on the specified data
258 * \param[in] req ahash request
259 * \param[in] buf data buffer on which to compute the hash
260 * \param[in] buf_len length of \buf on which to compute hash
262 * \retval 0 for success
263 * \retval negative errno on failure
265 int cfs_crypto_hash_update(struct ahash_request *req,
266 const void *buf, unsigned int buf_len)
268 struct scatterlist sl;
270 sg_init_one(&sl, (void *)buf, buf_len);
272 ahash_request_set_crypt(req, &sl, NULL, sl.length);
273 return crypto_ahash_update(req);
275 EXPORT_SYMBOL(cfs_crypto_hash_update);
278 * Finish hash calculation, copy hash digest to buffer, clean up hash descriptor
280 * \param[in] req ahash request
281 * \param[out] hash pointer to hash buffer to store hash digest
282 * \param[in,out] hash_len pointer to hash buffer size, if \a hash == NULL
283 * or hash_len == NULL only free \a hdesc instead
284 * of computing the hash
286 * \retval 0 for success
287 * \retval -EOVERFLOW if hash_len is too small for the hash digest
288 * \retval negative errno for other errors from lower layers
290 int cfs_crypto_hash_final(struct ahash_request *req,
291 unsigned char *hash, unsigned int *hash_len)
293 int size = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
296 if (!hash || !hash_len) {
300 if (*hash_len < size) {
305 ahash_request_set_crypt(req, NULL, hash, 0);
306 err = crypto_ahash_final(req);
310 crypto_free_ahash(crypto_ahash_reqtfm(req));
311 ahash_request_free(req);
315 EXPORT_SYMBOL(cfs_crypto_hash_final);
318 * Compute the speed of specified hash function
320 * Run a speed test on the given hash algorithm on buffer using a 1MB buffer
321 * size. This is a reasonable buffer size for Lustre RPCs, even if the actual
322 * RPC size is larger or smaller.
324 * The speed is stored internally in the cfs_crypto_hash_speeds[] array, and
325 * is available through the cfs_crypto_hash_speed() function.
327 * This function needs to stay the same as obd_t10_performance_test() so that
328 * the speeds are comparable.
330 * \param[in] hash_alg hash algorithm id (CFS_HASH_ALG_*)
331 * \param[in] buf data buffer on which to compute the hash
332 * \param[in] buf_len length of \buf on which to compute hash
334 static void cfs_crypto_performance_test(enum cfs_crypto_hash_alg hash_alg)
336 int buf_len = max(PAGE_SIZE, 1048576UL);
338 unsigned long start, end;
340 unsigned long bcount;
342 unsigned char hash[CFS_CRYPTO_HASH_DIGESTSIZE_MAX];
343 unsigned int hash_len = sizeof(hash);
345 page = alloc_page(GFP_KERNEL);
352 memset(buf, 0xAD, PAGE_SIZE);
355 for (start = jiffies, end = start + cfs_time_seconds(1) / 4,
356 bcount = 0; time_before(jiffies, end) && err == 0; bcount++) {
357 struct ahash_request *req;
360 req = cfs_crypto_hash_init(hash_alg, NULL, 0);
366 for (i = 0; i < buf_len / PAGE_SIZE; i++) {
367 err = cfs_crypto_hash_update_page(req, page, 0,
373 err = cfs_crypto_hash_final(req, hash, &hash_len);
381 cfs_crypto_hash_speeds[hash_alg] = err;
382 CDEBUG(D_INFO, "Crypto hash algorithm %s test error: rc = %d\n",
383 cfs_crypto_hash_name(hash_alg), err);
387 tmp = ((bcount * buf_len / jiffies_to_msecs(end - start)) *
388 1000) / (1024 * 1024);
389 cfs_crypto_hash_speeds[hash_alg] = (int)tmp;
390 CDEBUG(D_CONFIG, "Crypto hash algorithm %s speed = %d MB/s\n",
391 cfs_crypto_hash_name(hash_alg),
392 cfs_crypto_hash_speeds[hash_alg]);
397 * hash speed in Mbytes per second for valid hash algorithm
399 * Return the performance of the specified \a hash_alg that was
400 * computed using cfs_crypto_performance_test(). If the performance
401 * has not yet been computed, do that when it is first requested.
402 * That avoids computing the speed when it is not actually needed.
403 * To avoid competing threads computing the checksum speed at the
404 * same time, only compute a single checksum speed at one time.
406 * \param[in] hash_alg hash algorithm id (CFS_HASH_ALG_*)
408 * \retval positive speed of the hash function in MB/s
409 * \retval -ENOENT if \a hash_alg is unsupported
410 * \retval negative errno if \a hash_alg speed is unavailable
412 int cfs_crypto_hash_speed(enum cfs_crypto_hash_alg hash_alg)
414 if (hash_alg < CFS_HASH_ALG_MAX) {
415 if (unlikely(cfs_crypto_hash_speeds[hash_alg] == 0)) {
416 static DEFINE_MUTEX(crypto_hash_speed_mutex);
418 mutex_lock(&crypto_hash_speed_mutex);
419 if (cfs_crypto_hash_speeds[hash_alg] == 0)
420 cfs_crypto_performance_test(hash_alg);
421 mutex_unlock(&crypto_hash_speed_mutex);
423 return cfs_crypto_hash_speeds[hash_alg];
428 EXPORT_SYMBOL(cfs_crypto_hash_speed);
431 * Run the performance test for all hash algorithms.
433 * Run the cfs_crypto_performance_test() benchmark for some of the available
434 * hash functions at module load time. This can't be reliably done at runtime
435 * since the CPUs may be under load from thousands of connecting clients when
436 * the first client connects and the checksum speeds are needed.
438 * Since the setup cost and computation speed of various hash algorithms is
439 * a function of the buffer size (and possibly internal contention of offload
440 * engines), this speed only represents an estimate of the actual speed under
441 * actual usage, but is reasonable for comparing available algorithms.
443 * The actual speeds are available via cfs_crypto_hash_speed() for later
446 * \retval 0 on success
447 * \retval -ENOMEM if no memory is available for test buffer
449 static int cfs_crypto_test_hashes(void)
451 enum cfs_crypto_hash_alg hash_alg;
453 for (hash_alg = 1; hash_alg < CFS_HASH_ALG_SPEED_MAX; hash_alg++)
454 cfs_crypto_performance_test(hash_alg);
462 * Register available hash functions
466 int cfs_crypto_register(void)
468 request_module("crc32c");
470 if (cfs_crypto_adler32_register() == 0)
473 /* check all algorithms and do performance test */
474 cfs_crypto_test_hashes();
480 * Unregister previously registered hash functions
482 void cfs_crypto_unregister(void)
485 cfs_crypto_adler32_unregister();