Skip to content

Commit cb2a1dd

Browse files
hcahcahcahca
committed
s390/checksum: provide vector register variant of csum_partial()
Provide a faster variant of csum_partial() which uses vector registers instead of the cksm instruction. Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
1 parent 3a74f44 commit cb2a1dd

6 files changed

Lines changed: 187 additions & 16 deletions

File tree

‎arch/s390/include/asm/checksum.h‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -30,22 +30,7 @@ static inline __wsum cksm(const void *buff, int len, __wsum sum)
3030
return sum;
3131
}
3232

33-
/*
34-
* Computes the checksum of a memory block at buff, length len,
35-
* and adds in "sum" (32-bit).
36-
*
37-
* Returns a 32-bit number suitable for feeding into itself
38-
* or csum_tcpudp_magic.
39-
*
40-
* This function must be called with even lengths, except
41-
* for the last fragment, which may be odd.
42-
*
43-
* It's best to have buff aligned on a 32-bit boundary.
44-
*/
45-
static inline __wsum csum_partial(const void *buff, int len, __wsum sum)
46-
{
47-
return cksm(buff, len, sum);
48-
}
33+
__wsum csum_partial(const void *buff, int len, __wsum sum);
4934

5035
/*
5136
* Fold a partial checksum without adding pseudo headers.

‎arch/s390/include/asm/fpu-insn-asm.h‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,15 @@
521521
VMRL \vr1, \vr2, \vr3, 3
522522
.endm
523523

524+
/* VECTOR LOAD WITH LENGTH */
525+
.macro VLL v, gr, disp, base
526+
VX_NUM v1, \v
527+
GR_NUM b2, \base
528+
GR_NUM r3, \gr
529+
.word 0xE700 | ((v1&15) << 4) | r3
530+
.word (b2 << 12) | (\disp)
531+
MRXBOPC 0, 0x37, v1
532+
.endm
524533

525534
/* Vector integer instructions */
526535

@@ -534,6 +543,16 @@
534543
MRXBOPC 0, 0x68, v1, v2, v3
535544
.endm
536545

546+
/* VECTOR CHECKSUM */
547+
.macro VCKSM vr1, vr2, vr3
548+
VX_NUM v1, \vr1
549+
VX_NUM v2, \vr2
550+
VX_NUM v3, \vr3
551+
.word 0xE700 | ((v1&15) << 4) | (v2&15)
552+
.word ((v3&15) << 12)
553+
MRXBOPC 0, 0x66, v1, v2, v3
554+
.endm
555+
537556
/* VECTOR EXCLUSIVE OR */
538557
.macro VX vr1, vr2, vr3
539558
VX_NUM v1, \vr1

‎arch/s390/include/asm/fpu-insn.h‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,89 @@ static __always_inline void fpu_stfpc(unsigned int *fpc)
108108
: "memory");
109109
}
110110

111+
static __always_inline void fpu_vcksm(u8 v1, u8 v2, u8 v3)
112+
{
113+
asm volatile("VCKSM %[v1],%[v2],%[v3]"
114+
:
115+
: [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
116+
: "memory");
117+
}
118+
119+
#ifdef CONFIG_CC_IS_CLANG
120+
121+
static __always_inline void fpu_vl(u8 v1, const void *vxr)
122+
{
123+
instrument_read(vxr, sizeof(__vector128));
124+
asm volatile("\n"
125+
" la 1,%[vxr]\n"
126+
" VL %[v1],0,,1\n"
127+
:
128+
: [vxr] "R" (*(__vector128 *)vxr),
129+
[v1] "I" (v1)
130+
: "memory", "1");
131+
}
132+
133+
#else /* CONFIG_CC_IS_CLANG */
134+
135+
static __always_inline void fpu_vl(u8 v1, const void *vxr)
136+
{
137+
instrument_read(vxr, sizeof(__vector128));
138+
asm volatile("VL %[v1],%O[vxr],,%R[vxr]\n"
139+
:
140+
: [vxr] "Q" (*(__vector128 *)vxr),
141+
[v1] "I" (v1)
142+
: "memory");
143+
}
144+
145+
#endif /* CONFIG_CC_IS_CLANG */
146+
147+
static __always_inline u64 fpu_vlgvf(u8 v, u16 index)
148+
{
149+
u64 val;
150+
151+
asm volatile("VLGVF %[val],%[v],%[index]"
152+
: [val] "=d" (val)
153+
: [v] "I" (v), [index] "L" (index)
154+
: "memory");
155+
return val;
156+
}
157+
158+
#ifdef CONFIG_CC_IS_CLANG
159+
160+
static __always_inline void fpu_vll(u8 v1, u32 index, const void *vxr)
161+
{
162+
unsigned int size;
163+
164+
size = min(index + 1, sizeof(__vector128));
165+
instrument_read(vxr, size);
166+
asm volatile("\n"
167+
" la 1,%[vxr]\n"
168+
" VLL %[v1],%[index],0,1\n"
169+
:
170+
: [vxr] "R" (*(u8 *)vxr),
171+
[index] "d" (index),
172+
[v1] "I" (v1)
173+
: "memory", "1");
174+
}
175+
176+
#else /* CONFIG_CC_IS_CLANG */
177+
178+
static __always_inline void fpu_vll(u8 v1, u32 index, const void *vxr)
179+
{
180+
unsigned int size;
181+
182+
size = min(index + 1, sizeof(__vector128));
183+
instrument_read(vxr, size);
184+
asm volatile("VLL %[v1],%[index],%O[vxr],%R[vxr]\n"
185+
:
186+
: [vxr] "Q" (*(u8 *)vxr),
187+
[index] "d" (index),
188+
[v1] "I" (v1)
189+
: "memory");
190+
}
191+
192+
#endif /* CONFIG_CC_IS_CLANG */
193+
111194
#ifdef CONFIG_CC_IS_CLANG
112195

113196
#define fpu_vlm(_v1, _v3, _vxrs) \
@@ -148,6 +231,14 @@ static __always_inline void fpu_stfpc(unsigned int *fpc)
148231

149232
#endif /* CONFIG_CC_IS_CLANG */
150233

234+
static __always_inline void fpu_vlvgf(u8 v, u32 val, u16 index)
235+
{
236+
asm volatile("VLVGF %[v],%[val],%[index]"
237+
:
238+
: [v] "I" (v), [val] "d" (val), [index] "L" (index)
239+
: "memory");
240+
}
241+
151242
#ifdef CONFIG_CC_IS_CLANG
152243

153244
#define fpu_vstm(_v1, _v3, _vxrs) \
@@ -186,5 +277,13 @@ static __always_inline void fpu_stfpc(unsigned int *fpc)
186277

187278
#endif /* CONFIG_CC_IS_CLANG */
188279

280+
static __always_inline void fpu_vzero(u8 v)
281+
{
282+
asm volatile("VZERO %[v]"
283+
:
284+
: [v] "I" (v)
285+
: "memory");
286+
}
287+
189288
#endif /* __ASSEMBLY__ */
190289
#endif /* __ASM_S390_FPU_INSN_H */

‎arch/s390/include/asm/fpu-types.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,16 @@ struct kernel_fpu_##vxr_size { \
3232
__vector128 vxrs[vxr_size] __aligned(8); \
3333
}
3434

35+
KERNEL_FPU_STRUCT(8);
3536
KERNEL_FPU_STRUCT(16);
3637
KERNEL_FPU_STRUCT(32);
3738

3839
#define DECLARE_KERNEL_FPU_ONSTACK(vxr_size, name) \
3940
struct kernel_fpu_##vxr_size name __uninitialized
4041

42+
#define DECLARE_KERNEL_FPU_ONSTACK8(name) \
43+
DECLARE_KERNEL_FPU_ONSTACK(8, name)
44+
4145
#define DECLARE_KERNEL_FPU_ONSTACK16(name) \
4246
DECLARE_KERNEL_FPU_ONSTACK(16, name)
4347

‎arch/s390/lib/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#
55

66
lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o
7+
lib-y += csum-partial.o
78
obj-y += mem.o xor.o
89
lib-$(CONFIG_KPROBES) += probes.o
910
lib-$(CONFIG_UPROBES) += probes.o

‎arch/s390/lib/csum-partial.c‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
3+
#include <linux/export.h>
4+
#include <asm/checksum.h>
5+
#include <asm/fpu.h>
6+
7+
/*
8+
* Computes the checksum of a memory block at buff, length len,
9+
* and adds in "sum" (32-bit).
10+
*
11+
* Returns a 32-bit number suitable for feeding into itself
12+
* or csum_tcpudp_magic.
13+
*
14+
* This function must be called with even lengths, except
15+
* for the last fragment, which may be odd.
16+
*
17+
* It's best to have buff aligned on a 64-bit boundary.
18+
*/
19+
__wsum csum_partial(const void *buff, int len, __wsum sum)
20+
{
21+
DECLARE_KERNEL_FPU_ONSTACK8(vxstate);
22+
23+
if (!cpu_has_vx())
24+
return cksm(buff, len, sum);
25+
kernel_fpu_begin(&vxstate, KERNEL_VXR_V16V23);
26+
fpu_vlvgf(16, (__force u32)sum, 1);
27+
fpu_vzero(17);
28+
fpu_vzero(18);
29+
fpu_vzero(19);
30+
while (len >= 64) {
31+
fpu_vlm(20, 23, buff);
32+
fpu_vcksm(16, 20, 16);
33+
fpu_vcksm(17, 21, 17);
34+
fpu_vcksm(18, 22, 18);
35+
fpu_vcksm(19, 23, 19);
36+
buff += 64;
37+
len -= 64;
38+
}
39+
while (len >= 32) {
40+
fpu_vlm(20, 21, buff);
41+
fpu_vcksm(16, 20, 16);
42+
fpu_vcksm(17, 21, 17);
43+
buff += 32;
44+
len -= 32;
45+
}
46+
while (len >= 16) {
47+
fpu_vl(20, buff);
48+
fpu_vcksm(16, 20, 16);
49+
buff += 16;
50+
len -= 16;
51+
}
52+
if (len) {
53+
fpu_vll(20, len - 1, buff);
54+
fpu_vcksm(16, 20, 16);
55+
}
56+
fpu_vcksm(18, 19, 18);
57+
fpu_vcksm(16, 17, 16);
58+
fpu_vcksm(16, 18, 16);
59+
sum = (__force __wsum)fpu_vlgvf(16, 1);
60+
kernel_fpu_end(&vxstate, KERNEL_VXR_V16V23);
61+
return sum;
62+
}
63+
EXPORT_SYMBOL(csum_partial);

0 commit comments

Comments
 (0)