3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 only,
7 * as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License version 2 for more details (a copy is included
13 * in the LICENSE file that accompanied this code).
15 * You should have received a copy of the GNU General Public License
16 * version 2 along with this program; If not, see http://www.gnu.org/licenses
18 * Please visit http://www.xyratex.com/contact if you need additional
19 * information or have any questions.
25 * Copyright 2012 Xyratex Technology Limited
27 * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
29 * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
30 * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
32 * http://www.intel.com/products/processor/manuals/
33 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
34 * Volume 2B: Instruction Set Reference, N-Z
36 * Authors: Gregory Prestas <Gregory_Prestas@us.xyratex.com>
37 * Alexander Boyko <Alexander_Boyko@xyratex.com>
40 /* gcc 4.1.2 does not support pclmulqdq instruction
41 * Use macro defenition from linux kernel 2.6.38 */
43 #define REG_NUM_INVALID 100
44 .macro R32_NUM opd r32
45 \opd = REG_NUM_INVALID
72 .macro XMM_NUM opd xmm
73 \opd = REG_NUM_INVALID
128 .macro PFX_REX opd1 opd2 W=0
129 .if ((\opd1 | \opd2) & 8) || \W
130 .byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
134 .macro MODRM mod opd1 opd2
135 .byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
138 .macro PCLMULQDQ imm8 xmm1 xmm2
139 XMM_NUM clmul_opd1 \xmm1
140 XMM_NUM clmul_opd2 \xmm2
142 PFX_REX clmul_opd1 clmul_opd2
143 .byte 0x0f, 0x3a, 0x44
144 MODRM 0xc0 clmul_opd1 clmul_opd2
148 .macro PEXTRD imm8 xmm1 reg1
149 XMM_NUM extrd_opd2 \xmm1
150 R32_NUM extrd_opd1 \reg1
152 PFX_REX extrd_opd1 extrd_opd2
153 .byte 0x0f, 0x3a, 0x16
154 MODRM 0xc0 extrd_opd1 extrd_opd2
160 * [x4*128+32 mod P(x) << 32)]' << 1 = 0x154442bd4
161 * #define CONSTANT_R1 0x154442bd4LL
163 * [(x4*128-32 mod P(x) << 32)]' << 1 = 0x1c6e41596
164 * #define CONSTANT_R2 0x1c6e41596LL
167 .octa 0x00000001c6e415960000000154442bd4
169 * [(x128+32 mod P(x) << 32)]' << 1 = 0x1751997d0
170 * #define CONSTANT_R3 0x1751997d0LL
172 * [(x128-32 mod P(x) << 32)]' << 1 = 0x0ccaa009e
173 * #define CONSTANT_R4 0x0ccaa009eLL
176 .octa 0x00000000ccaa009e00000001751997d0
178 * [(x64 mod P(x) << 32)]' << 1 = 0x163cd6124
179 * #define CONSTANT_R5 0x163cd6124LL
182 .octa 0x00000000000000000000000163cd6124
184 .octa 0x000000000000000000000000FFFFFFFF
186 * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
188 * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))` = 0x1F7011641LL
189 * #define CONSTANT_RU 0x1F7011641LL
192 .octa 0x00000001F701164100000001DB710641
194 #define CONSTANT %xmm0
211 * BUF - buffer (16 bytes aligned)
212 * LEN - sizeof buffer (16 bytes aligned), LEN should be grater than 63
213 * CRC - initial crc32
215 * uint crc32_pclmul_le_16(unsigned char const *buffer,
216 * size_t len, uint crc32)
218 .globl crc32_pclmul_le_16
220 crc32_pclmul_le_16:/* buffer and buffer size are 16 bytes aligned */
222 movdqa 0x10(BUF), %xmm2
223 movdqa 0x20(BUF), %xmm3
224 movdqa 0x30(BUF), %xmm4
230 /* This is for position independed code(-fPIC) support for 32bit */
239 movdqa .Lconstant_R2R1(%rip), CONSTANT
241 movdqa .Lconstant_R2R1 - delta(%ecx), CONSTANT
244 loop_64:/* 64 bytes Full cache line folding */
245 prefetchnta 0x40(BUF)
252 PCLMULQDQ 00, CONSTANT, %xmm1
253 PCLMULQDQ 00, CONSTANT, %xmm2
254 PCLMULQDQ 00, CONSTANT, %xmm3
256 PCLMULQDQ 00, CONSTANT, %xmm4
258 PCLMULQDQ 0x11, CONSTANT, %xmm5
259 PCLMULQDQ 0x11, CONSTANT, %xmm6
260 PCLMULQDQ 0x11, CONSTANT, %xmm7
262 PCLMULQDQ 0x11, CONSTANT, %xmm8
270 /* xmm8 unsupported for x32 */
272 PCLMULQDQ 00, CONSTANT, %xmm4
273 PCLMULQDQ 0x11, CONSTANT, %xmm5
278 pxor 0x10(BUF), %xmm2
279 pxor 0x20(BUF), %xmm3
280 pxor 0x30(BUF), %xmm4
286 less_64:/* Folding cache line into 128bit */
288 movdqa .Lconstant_R4R3(%rip), CONSTANT
290 movdqa .Lconstant_R4R3 - delta(%ecx), CONSTANT
295 PCLMULQDQ 0x00, CONSTANT, %xmm1
296 PCLMULQDQ 0x11, CONSTANT, %xmm5
301 PCLMULQDQ 0x00, CONSTANT, %xmm1
302 PCLMULQDQ 0x11, CONSTANT, %xmm5
307 PCLMULQDQ 0x00, CONSTANT, %xmm1
308 PCLMULQDQ 0x11, CONSTANT, %xmm5
314 loop_16:/* Folding rest buffer into 128bit */
316 PCLMULQDQ 0x00, CONSTANT, %xmm1
317 PCLMULQDQ 0x11, CONSTANT, %xmm5
326 /* perform the last 64 bit fold, also adds 32 zeroes
327 * to the input stream */
328 PCLMULQDQ 0x01, %xmm1, CONSTANT /* R4 * xmm1.low */
332 /* final 32-bit fold */
335 movdqa .Lconstant_R5(%rip), CONSTANT
336 movdqa .Lconstant_mask32(%rip), %xmm3
338 movdqa .Lconstant_R5 - delta(%ecx), CONSTANT
339 movdqa .Lconstant_mask32 - delta(%ecx), %xmm3
343 PCLMULQDQ 0x00, CONSTANT, %xmm1
346 /* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
348 movdqa .Lconstant_RUpoly(%rip), CONSTANT
350 movdqa .Lconstant_RUpoly - delta(%ecx), CONSTANT
354 PCLMULQDQ 0x10, CONSTANT, %xmm1
356 PCLMULQDQ 0x00, CONSTANT, %xmm1
358 PEXTRD 0x01, %xmm1, %eax