LU-3570 libcfs: accelerate crc32c with pclmulqdq

[fs/lustre-release.git] / libcfs / libcfs / linux / linux-crypto.c

diff --git a/libcfs/libcfs/linux/linux-crypto.c b/libcfs/libcfs/linux/linux-crypto.c
index 11caba4..8398e9e 100644 (file)
--- a/libcfs/libcfs/linux/linux-crypto.c
+++ b/libcfs/libcfs/linux/linux-crypto.c
@@ -36,73 +36,6 @@
  */
 static int cfs_crypto_hash_speeds[CFS_HASH_ALG_MAX];
 
  */
 static int cfs_crypto_hash_speeds[CFS_HASH_ALG_MAX];
 

-
-#ifndef HAVE_STRUCT_HASH_DESC
-/** 2.6.18 kernel have no crypto_hash function
- *  this part was copied from lustre_compat25.h */
-#define crypto_hash     crypto_tfm
-struct hash_desc {
-       struct crypto_hash      *tfm;
-       unsigned int            flags;
-};
-
-static inline
-struct crypto_hash *crypto_alloc_hash(const char *alg, unsigned int type,
-                                     unsigned int mask)
-{
-       return crypto_alloc_tfm(alg, 0);
-}
-
-static inline void crypto_free_hash(struct crypto_hash *tfm)
-{
-       crypto_free_tfm(tfm);
-}
-
-static inline int crypto_hash_init(struct hash_desc *desc)
-{
-       crypto_digest_init(desc->tfm);
-       return 0;
-}
-
-static inline int crypto_hash_update(struct hash_desc *desc,
-                                    struct scatterlist *sg,
-                                    unsigned int nbytes)
-{
-       if (desc->tfm->crt_digest.dit_update == NULL)
-               return -1;
-
-       LASSERT(nbytes == sg->length);
-       crypto_digest_update(desc->tfm, sg, 1);
-
-       return 0;
-}
-
-static inline int crypto_hash_digest(struct hash_desc *desc,
-                                    struct scatterlist *sg,
-                                    unsigned int nbytes, unsigned char *out)
-{
-       crypto_hash_update(desc, sg, nbytes);
-       crypto_digest_final(desc->tfm, out);
-       return 0;
-}
-
-static inline int crypto_hash_final(struct hash_desc *desc, unsigned char *out)
-{
-       crypto_digest_final(desc->tfm, out);
-       return 0;
-}
-
-static inline struct crypto_tfm *crypto_hash_tfm(struct crypto_hash *tfm)
-{
-       return tfm;
-}
-
-#define crypto_hash_setkey(tfm, key, keylen) \
-               crypto_digest_setkey(tfm, key, keylen)
-#define crypto_hash_digestsize(tfm)  crypto_tfm_alg_digestsize(tfm)
-#define crypto_hash_blocksize(tfm)   crypto_tfm_alg_blocksize(tfm)
-#endif
-

 static int cfs_crypto_hash_alloc(unsigned char alg_id,
                                 const struct cfs_crypto_hash_type **type,
                                 struct hash_desc *desc, unsigned char *key,
 static int cfs_crypto_hash_alloc(unsigned char alg_id,
                                 const struct cfs_crypto_hash_type **type,
                                 struct hash_desc *desc, unsigned char *key,


@@ -130,15 +63,6 @@ static int cfs_crypto_hash_alloc(unsigned char alg_id,
 
        desc->flags = 0;
 
 
        desc->flags = 0;
 

-       /** Shash have different logic for initialization then digest
-        * shash: crypto_hash_setkey, crypto_hash_init
-        * digest: crypto_digest_init, crypto_digest_setkey
-        * Skip this function for digest, because we use shash logic at
-        * cfs_crypto_hash_alloc.
-        */
-#ifndef HAVE_STRUCT_SHASH_ALG
-       crypto_hash_init(desc);
-#endif

        if (key != NULL) {
                err = crypto_hash_setkey(desc->tfm, key, key_len);
        } else if ((*type)->cht_key != 0) {
        if (key != NULL) {
                err = crypto_hash_setkey(desc->tfm, key, key_len);
        } else if ((*type)->cht_key != 0) {


@@ -157,11 +81,7 @@ static int cfs_crypto_hash_alloc(unsigned char alg_id,
               (crypto_hash_tfm(desc->tfm))->__crt_alg->cra_driver_name,
               cfs_crypto_hash_speeds[alg_id]);
 
               (crypto_hash_tfm(desc->tfm))->__crt_alg->cra_driver_name,
               cfs_crypto_hash_speeds[alg_id]);
 

-#ifdef HAVE_STRUCT_SHASH_ALG

        return crypto_hash_init(desc);
        return crypto_hash_init(desc);

-#else
-       return 0;
-#endif

 }
 
 int cfs_crypto_hash_digest(unsigned char alg_id,
 }
 
 int cfs_crypto_hash_digest(unsigned char alg_id,


@@ -205,14 +125,14 @@ struct cfs_crypto_hash_desc *
        int                  err;
        const struct cfs_crypto_hash_type       *type;
 
        int                  err;
        const struct cfs_crypto_hash_type       *type;
 

-       hdesc = cfs_alloc(sizeof(*hdesc), 0);
+       hdesc = kmalloc(sizeof(*hdesc), 0);

        if (hdesc == NULL)
                return ERR_PTR(-ENOMEM);
 
        err = cfs_crypto_hash_alloc(alg_id, &type, hdesc, key, key_len);
 
        if (err) {
        if (hdesc == NULL)
                return ERR_PTR(-ENOMEM);
 
        err = cfs_crypto_hash_alloc(alg_id, &type, hdesc, key, key_len);
 
        if (err) {

-               cfs_free(hdesc);
+               kfree(hdesc);

                return ERR_PTR(err);
        }
        return (struct cfs_crypto_hash_desc *)hdesc;
                return ERR_PTR(err);
        }
        return (struct cfs_crypto_hash_desc *)hdesc;


@@ -220,7 +140,7 @@ struct cfs_crypto_hash_desc *
 EXPORT_SYMBOL(cfs_crypto_hash_init);
 
 int cfs_crypto_hash_update_page(struct cfs_crypto_hash_desc *hdesc,
 EXPORT_SYMBOL(cfs_crypto_hash_init);
 
 int cfs_crypto_hash_update_page(struct cfs_crypto_hash_desc *hdesc,

-                               cfs_page_t *page, unsigned int offset,
+                               struct page *page, unsigned int offset,

                                unsigned int len)
 {
        struct scatterlist sl;
                                unsigned int len)
 {
        struct scatterlist sl;


@@ -252,7 +172,7 @@ int cfs_crypto_hash_final(struct cfs_crypto_hash_desc *hdesc,
 
        if (hash_len == NULL) {
                crypto_free_hash(((struct hash_desc *)hdesc)->tfm);
 
        if (hash_len == NULL) {
                crypto_free_hash(((struct hash_desc *)hdesc)->tfm);

-               cfs_free(hdesc);
+               kfree(hdesc);

                return 0;
        }
        if (hash == NULL || *hash_len < size) {
                return 0;
        }
        if (hash == NULL || *hash_len < size) {


@@ -266,7 +186,7 @@ int cfs_crypto_hash_final(struct cfs_crypto_hash_desc *hdesc,
                return err;
        }
        crypto_free_hash(((struct hash_desc *)hdesc)->tfm);
                return err;
        }
        crypto_free_hash(((struct hash_desc *)hdesc)->tfm);

-       cfs_free(hdesc);
+       kfree(hdesc);

        return err;
 }
 EXPORT_SYMBOL(cfs_crypto_hash_final);
        return err;
 }
 EXPORT_SYMBOL(cfs_crypto_hash_final);


@@ -301,7 +221,7 @@ static void cfs_crypto_performance_test(unsigned char alg_id,
                       1000) / (1024 * 1024);
                cfs_crypto_hash_speeds[alg_id] = (int)tmp;
        }
                       1000) / (1024 * 1024);
                cfs_crypto_hash_speeds[alg_id] = (int)tmp;
        }

-       CDEBUG(D_INFO, "Crypto hash algorithm %s speed = %d MB/s\n",
+       CDEBUG(D_CONFIG, "Crypto hash algorithm %s speed = %d MB/s\n",

               cfs_crypto_hash_name(alg_id), cfs_crypto_hash_speeds[alg_id]);
 }
 
               cfs_crypto_hash_name(alg_id), cfs_crypto_hash_speeds[alg_id]);
 }
 


@@ -326,7 +246,7 @@ static int cfs_crypto_test_hashes(void)
         * kmalloc size for 2.6.18 kernel is 128K */
        unsigned int        data_len = 1 * 128 * 1024;
 
         * kmalloc size for 2.6.18 kernel is 128K */
        unsigned int        data_len = 1 * 128 * 1024;
 

-       data = cfs_alloc(data_len, 0);
+       data = kmalloc(data_len, 0);

        if (data == NULL) {
                CERROR("Failed to allocate mem\n");
                return -ENOMEM;
        if (data == NULL) {
                CERROR("Failed to allocate mem\n");
                return -ENOMEM;


@@ -338,40 +258,63 @@ static int cfs_crypto_test_hashes(void)
        for (i = 0; i < CFS_HASH_ALG_MAX; i++)
                cfs_crypto_performance_test(i, data, data_len);
 
        for (i = 0; i < CFS_HASH_ALG_MAX; i++)
                cfs_crypto_performance_test(i, data, data_len);
 

-       cfs_free(data);
+       kfree(data);

        return 0;
 }
 
        return 0;
 }
 

-static int crc32, adler32;
+static int adler32;

 
 

+#ifdef HAVE_CRC32
+static int crc32;
+#endif

 #ifdef HAVE_PCLMULQDQ
 #ifdef HAVE_PCLMULQDQ

+#ifdef NEED_CRC32_ACCEL

 static int crc32pclmul;
 #endif
 static int crc32pclmul;
 #endif

+#ifdef NEED_CRC32C_ACCEL
+static int crc32c_pclmul;
+#endif
+#endif

 
 int cfs_crypto_register(void)
 {
 
 int cfs_crypto_register(void)
 {

-       crc32 = cfs_crypto_crc32_register();
+       request_module("crc32c");
+

        adler32 = cfs_crypto_adler32_register();
 
        adler32 = cfs_crypto_adler32_register();
 

+#ifdef HAVE_CRC32
+       crc32 = cfs_crypto_crc32_register();
+#endif

 #ifdef HAVE_PCLMULQDQ
 #ifdef HAVE_PCLMULQDQ

+#ifdef NEED_CRC32_ACCEL

        crc32pclmul = cfs_crypto_crc32_pclmul_register();
 #endif
        crc32pclmul = cfs_crypto_crc32_pclmul_register();
 #endif

-
+#ifdef NEED_CRC32C_ACCEL
+       crc32c_pclmul = cfs_crypto_crc32c_pclmul_register();
+#endif
+#endif

        /* check all algorithms and do performance test */
        cfs_crypto_test_hashes();
        return 0;
 }
 void cfs_crypto_unregister(void)
 {
        /* check all algorithms and do performance test */
        cfs_crypto_test_hashes();
        return 0;
 }
 void cfs_crypto_unregister(void)
 {

-       if (crc32 == 0)
-               cfs_crypto_crc32_unregister();

        if (adler32 == 0)
                cfs_crypto_adler32_unregister();
 
        if (adler32 == 0)
                cfs_crypto_adler32_unregister();
 

+#ifdef HAVE_CRC32
+       if (crc32 == 0)
+               cfs_crypto_crc32_unregister();
+#endif

 #ifdef HAVE_PCLMULQDQ
 #ifdef HAVE_PCLMULQDQ

+#ifdef NEED_CRC32_ACCEL

        if (crc32pclmul == 0)
                cfs_crypto_crc32_pclmul_unregister();
 #endif
        if (crc32pclmul == 0)
                cfs_crypto_crc32_pclmul_unregister();
 #endif

-
+#ifdef NEED_CRC32C_ACCEL
+       if (crc32c_pclmul == 0)
+               cfs_crypto_crc32c_pclmul_unregister();
+#endif
+#endif

        return;
 }
        return;
 }