Whamcloud - gitweb
LU-3570 libcfs: accelerate crc32c with pclmulqdq
[fs/lustre-release.git] / libcfs / libcfs / linux / linux-crypto.c
index a15938a..8398e9e 100644 (file)
@@ -23,6 +23,8 @@
 
 /*
  * Copyright 2012 Xyratex Technology Limited
+ *
+ * Copyright (c) 2012, Intel Corporation.
  */
 
 #include <linux/crypto.h>
  */
 static int cfs_crypto_hash_speeds[CFS_HASH_ALG_MAX];
 
-
-#ifndef HAVE_STRUCT_HASH_DESC
-/** 2.6.18 kernel have no crypto_hash function
- *  this part was copied from lustre_compat25.h */
-#define crypto_hash     crypto_tfm
-struct hash_desc {
-       struct crypto_hash      *tfm;
-       unsigned int            flags;
-};
-
-static inline
-struct crypto_hash *crypto_alloc_hash(const char *alg, unsigned int type,
-                                     unsigned int mask)
-{
-       return crypto_alloc_tfm(alg, 0);
-}
-
-static inline void crypto_free_hash(struct crypto_hash *tfm)
-{
-       crypto_free_tfm(tfm);
-}
-
-static inline int crypto_hash_init(struct hash_desc *desc)
-{
-       crypto_digest_init(desc->tfm);
-       return 0;
-}
-
-static inline int crypto_hash_update(struct hash_desc *desc,
-                                    struct scatterlist *sg,
-                                    unsigned int nbytes)
-{
-       if (desc->tfm->crt_digest.dit_update == NULL)
-               return -1;
-
-       LASSERT(nbytes == sg->length);
-       crypto_digest_update(desc->tfm, sg, 1);
-
-       return 0;
-}
-
-static inline int crypto_hash_digest(struct hash_desc *desc,
-                                    struct scatterlist *sg,
-                                    unsigned int nbytes, unsigned char *out)
-{
-       crypto_hash_update(desc, sg, nbytes);
-       crypto_digest_final(desc->tfm, out);
-       return 0;
-}
-
-static inline int crypto_hash_final(struct hash_desc *desc, unsigned char *out)
-{
-       crypto_digest_final(desc->tfm, out);
-       return 0;
-}
-
-static inline struct crypto_tfm *crypto_hash_tfm(struct crypto_hash *tfm)
-{
-       return tfm;
-}
-
-#define crypto_hash_setkey(tfm, key, keylen) \
-               crypto_digest_setkey(tfm, key, keylen)
-#define crypto_hash_digestsize(tfm)  crypto_tfm_alg_digestsize(tfm)
-#define crypto_hash_blocksize(tfm)   crypto_tfm_alg_blocksize(tfm)
-#endif
-
 static int cfs_crypto_hash_alloc(unsigned char alg_id,
                                 const struct cfs_crypto_hash_type **type,
                                 struct hash_desc *desc, unsigned char *key,
@@ -128,15 +63,6 @@ static int cfs_crypto_hash_alloc(unsigned char alg_id,
 
        desc->flags = 0;
 
-       /** Shash have different logic for initialization then digest
-        * shash: crypto_hash_setkey, crypto_hash_init
-        * digest: crypto_digest_init, crypto_digest_setkey
-        * Skip this function for digest, because we use shash logic at
-        * cfs_crypto_hash_alloc.
-        */
-#ifndef HAVE_STRUCT_SHASH_ALG
-       crypto_hash_init(desc);
-#endif
        if (key != NULL) {
                err = crypto_hash_setkey(desc->tfm, key, key_len);
        } else if ((*type)->cht_key != 0) {
@@ -151,15 +77,11 @@ static int cfs_crypto_hash_alloc(unsigned char alg_id,
        }
 
        CDEBUG(D_INFO, "Using crypto hash: %s (%s) speed %d MB/s\n",
-              crypto_tfm_alg_name(crypto_hash_tfm(desc->tfm)),
-              crypto_tfm_alg_driver_name(crypto_hash_tfm(desc->tfm)),
+              (crypto_hash_tfm(desc->tfm))->__crt_alg->cra_name,
+              (crypto_hash_tfm(desc->tfm))->__crt_alg->cra_driver_name,
               cfs_crypto_hash_speeds[alg_id]);
 
-#ifdef HAVE_STRUCT_SHASH_ALG
        return crypto_hash_init(desc);
-#else
-       return 0;
-#endif
 }
 
 int cfs_crypto_hash_digest(unsigned char alg_id,
@@ -167,7 +89,7 @@ int cfs_crypto_hash_digest(unsigned char alg_id,
                           unsigned char *key, unsigned int key_len,
                           unsigned char *hash, unsigned int *hash_len)
 {
-       struct scatterlist      sl = {0};
+       struct scatterlist      sl;
        struct hash_desc        hdesc;
        int                     err;
        const struct cfs_crypto_hash_type       *type;
@@ -184,7 +106,7 @@ int cfs_crypto_hash_digest(unsigned char alg_id,
                crypto_free_hash(hdesc.tfm);
                return -ENOSPC;
        }
-       sg_set_buf(&sl, (void *)buf, buf_len);
+       sg_init_one(&sl, (void *)buf, buf_len);
 
        hdesc.flags = 0;
        err = crypto_hash_digest(&hdesc, &sl, sl.length, hash);
@@ -203,14 +125,14 @@ struct cfs_crypto_hash_desc *
        int                  err;
        const struct cfs_crypto_hash_type       *type;
 
-       hdesc = cfs_alloc(sizeof(*hdesc), 0);
+       hdesc = kmalloc(sizeof(*hdesc), 0);
        if (hdesc == NULL)
                return ERR_PTR(-ENOMEM);
 
        err = cfs_crypto_hash_alloc(alg_id, &type, hdesc, key, key_len);
 
        if (err) {
-               cfs_free(hdesc);
+               kfree(hdesc);
                return ERR_PTR(err);
        }
        return (struct cfs_crypto_hash_desc *)hdesc;
@@ -218,11 +140,12 @@ struct cfs_crypto_hash_desc *
 EXPORT_SYMBOL(cfs_crypto_hash_init);
 
 int cfs_crypto_hash_update_page(struct cfs_crypto_hash_desc *hdesc,
-                               cfs_page_t *page, unsigned int offset,
+                               struct page *page, unsigned int offset,
                                unsigned int len)
 {
-       struct scatterlist      sl = {0};
+       struct scatterlist sl;
 
+       sg_init_table(&sl, 1);
        sg_set_page(&sl, page, len, offset & ~CFS_PAGE_MASK);
 
        return crypto_hash_update((struct hash_desc *)hdesc, &sl, sl.length);
@@ -232,9 +155,9 @@ EXPORT_SYMBOL(cfs_crypto_hash_update_page);
 int cfs_crypto_hash_update(struct cfs_crypto_hash_desc *hdesc,
                           const void *buf, unsigned int buf_len)
 {
-       struct scatterlist      sl = {0};
+       struct scatterlist sl;
 
-       sg_set_buf(&sl, (void *)buf, buf_len);
+       sg_init_one(&sl, (void *)buf, buf_len);
 
        return crypto_hash_update((struct hash_desc *)hdesc, &sl, sl.length);
 }
@@ -249,6 +172,7 @@ int cfs_crypto_hash_final(struct cfs_crypto_hash_desc *hdesc,
 
        if (hash_len == NULL) {
                crypto_free_hash(((struct hash_desc *)hdesc)->tfm);
+               kfree(hdesc);
                return 0;
        }
        if (hash == NULL || *hash_len < size) {
@@ -262,6 +186,7 @@ int cfs_crypto_hash_final(struct cfs_crypto_hash_desc *hdesc,
                return err;
        }
        crypto_free_hash(((struct hash_desc *)hdesc)->tfm);
+       kfree(hdesc);
        return err;
 }
 EXPORT_SYMBOL(cfs_crypto_hash_final);
@@ -296,7 +221,7 @@ static void cfs_crypto_performance_test(unsigned char alg_id,
                       1000) / (1024 * 1024);
                cfs_crypto_hash_speeds[alg_id] = (int)tmp;
        }
-       CDEBUG(D_INFO, "Crypto hash algorithm %s speed = %d MB/s\n",
+       CDEBUG(D_CONFIG, "Crypto hash algorithm %s speed = %d MB/s\n",
               cfs_crypto_hash_name(alg_id), cfs_crypto_hash_speeds[alg_id]);
 }
 
@@ -321,7 +246,7 @@ static int cfs_crypto_test_hashes(void)
         * kmalloc size for 2.6.18 kernel is 128K */
        unsigned int        data_len = 1 * 128 * 1024;
 
-       data = cfs_alloc(data_len, 0);
+       data = kmalloc(data_len, 0);
        if (data == NULL) {
                CERROR("Failed to allocate mem\n");
                return -ENOMEM;
@@ -333,26 +258,63 @@ static int cfs_crypto_test_hashes(void)
        for (i = 0; i < CFS_HASH_ALG_MAX; i++)
                cfs_crypto_performance_test(i, data, data_len);
 
-       cfs_free(data);
+       kfree(data);
        return 0;
 }
 
-static int crc32, adler32;
+static int adler32;
+
+#ifdef HAVE_CRC32
+static int crc32;
+#endif
+#ifdef HAVE_PCLMULQDQ
+#ifdef NEED_CRC32_ACCEL
+static int crc32pclmul;
+#endif
+#ifdef NEED_CRC32C_ACCEL
+static int crc32c_pclmul;
+#endif
+#endif
 
 int cfs_crypto_register(void)
 {
-       crc32 = cfs_crypto_crc32_register();
+       request_module("crc32c");
+
        adler32 = cfs_crypto_adler32_register();
 
-       /* check all algorithms and do perfermance test */
+#ifdef HAVE_CRC32
+       crc32 = cfs_crypto_crc32_register();
+#endif
+#ifdef HAVE_PCLMULQDQ
+#ifdef NEED_CRC32_ACCEL
+       crc32pclmul = cfs_crypto_crc32_pclmul_register();
+#endif
+#ifdef NEED_CRC32C_ACCEL
+       crc32c_pclmul = cfs_crypto_crc32c_pclmul_register();
+#endif
+#endif
+       /* check all algorithms and do performance test */
        cfs_crypto_test_hashes();
        return 0;
 }
 void cfs_crypto_unregister(void)
 {
-       if (crc32 == 0)
-               cfs_crypto_crc32_unregister();
        if (adler32 == 0)
                cfs_crypto_adler32_unregister();
+
+#ifdef HAVE_CRC32
+       if (crc32 == 0)
+               cfs_crypto_crc32_unregister();
+#endif
+#ifdef HAVE_PCLMULQDQ
+#ifdef NEED_CRC32_ACCEL
+       if (crc32pclmul == 0)
+               cfs_crypto_crc32_pclmul_unregister();
+#endif
+#ifdef NEED_CRC32C_ACCEL
+       if (crc32c_pclmul == 0)
+               cfs_crypto_crc32c_pclmul_unregister();
+#endif
+#endif
        return;
 }