LU-3570 libcfs: accelerate crc32c with pclmulqdq

[fs/lustre-release.git] / libcfs / libcfs / user-crypto.c

diff --git a/libcfs/libcfs/user-crypto.c b/libcfs/libcfs/user-crypto.c
index 66d0bae..dd4c5dd 100644 (file)
--- a/libcfs/libcfs/user-crypto.c
+++ b/libcfs/libcfs/user-crypto.c
@@ -31,6 +31,7 @@
 
 #include <libcfs/libcfs.h>
 #include <libcfs/posix/posix-crypto.h>
+#include <libcfs/user-crypto.h>
 
 static int cfs_crypto_hash_speeds[CFS_HASH_ALG_MAX];
 
@@ -89,8 +90,21 @@ static int adler_wrapper(void *ctx, const unsigned char *p,
        return 0;
 }
 
+#if defined(HAVE_PCLMULQDQ) && defined(NEED_CRC32_ACCEL)
+static int crc32_pclmul_wrapper(void *ctx, const unsigned char *p,
+                               unsigned int len)
+{
+       unsigned int cksum = *(unsigned int *)ctx;
+
+       cksum = crc32_pclmul_le(cksum, p, len);
+
+       *(unsigned int *)ctx = cksum;
+       return 0;
+}
+#endif
+
 static int start_generic(void *ctx, unsigned char *key,
-                                        unsigned int key_len)
+                        unsigned int key_len)
 {
        const struct cfs_crypto_hash_type       *type;
        struct hash_desc        *hd = container_of(ctx, struct hash_desc,
@@ -111,7 +125,7 @@ static int start_generic(void *ctx, unsigned char *key,
 }
 
 static int final_generic(void *ctx, unsigned char *hash,
-                                        unsigned int hash_len)
+                        unsigned int hash_len)
 {
        const struct cfs_crypto_hash_type       *type;
        struct hash_desc        *hd = container_of(ctx, struct hash_desc,
@@ -142,7 +156,18 @@ static struct __hash_alg crypto_hash[] = {
                                           .update = adler_wrapper,
                                           .start = start_generic,
                                           .final = final_generic,
-                                          .fini = NULL} };
+                                          .fini = NULL},
+#if defined(HAVE_PCLMULQDQ) && defined(NEED_CRC32_ACCEL)
+                                         {.ha_id = CFS_HASH_ALG_CRC32,
+                                          .ha_ctx_size = sizeof(unsigned int),
+                                          .ha_priority = 100,
+                                          .init = crc32_pclmul_init,
+                                          .update = crc32_pclmul_wrapper,
+                                          .start = start_generic,
+                                          .final = final_generic,
+                                          .fini = NULL},
+#endif
+                                       };
 
 /**
  * Go through hashes to find the hash with  max priority
@@ -189,7 +214,7 @@ struct cfs_crypto_hash_desc
                return ERR_PTR(-ENODEV);
        }
 
-       hdesc = cfs_alloc(sizeof(*hdesc) + ha->ha_ctx_size, 0);
+       hdesc = kmalloc(sizeof(*hdesc) + ha->ha_ctx_size, 0);
        if (hdesc == NULL)
                return ERR_PTR(-ENOMEM);
 
@@ -200,7 +225,7 @@ struct cfs_crypto_hash_desc
                if (err == 0) {
                        return (struct cfs_crypto_hash_desc *) hdesc;
                } else {
-                       cfs_free(hdesc);
+                       kfree(hdesc);
                        return ERR_PTR(err);
                }
        }
@@ -216,7 +241,7 @@ int cfs_crypto_hash_update(struct cfs_crypto_hash_desc *desc, const void *buf,
 }
 
 int cfs_crypto_hash_update_page(struct cfs_crypto_hash_desc *desc,
-                               cfs_page_t *page, unsigned int offset,
+                               struct page *page, unsigned int offset,
                                unsigned int len)
 {
        const void *p = page->addr + offset;
@@ -237,7 +262,7 @@ int cfs_crypto_hash_final(struct cfs_crypto_hash_desc *desc,
        int     err;
 
        if (hash_len == NULL) {
-               cfs_free(d);
+               kfree(d);
                return 0;
        }
        if (hash == NULL || *hash_len < size) {
@@ -249,7 +274,7 @@ int cfs_crypto_hash_final(struct cfs_crypto_hash_desc *desc,
        err = d->hd_hash->final(d->hd_ctx, hash, *hash_len);
        if (err == 0) {
                  /* If get final digest success free hash descriptor */
-                 cfs_free(d);
+                 kfree(d);
        }
 
        return err;
@@ -345,7 +370,7 @@ static int cfs_crypto_test_hashes(void)
        unsigned char      *data;
        unsigned int        j, data_len = 1024 * 1024;
 
-       data = cfs_alloc(data_len, 0);
+       data = kmalloc(data_len, 0);
        if (data == NULL) {
                CERROR("Failed to allocate mem\n");
                return -ENOMEM;
@@ -356,7 +381,7 @@ static int cfs_crypto_test_hashes(void)
        for (i = 0; i < CFS_HASH_ALG_MAX; i++)
                cfs_crypto_performance_test(i, data, data_len);
 
-       cfs_free(data);
+       kfree(data);
        return 0;
 }