-
Notifications
You must be signed in to change notification settings - Fork 493
/
0064-AArch64-Check-kernel-version-for-SVE-ifuncs.patch
153 lines (142 loc) · 5.39 KB
/
0064-AArch64-Check-kernel-version-for-SVE-ifuncs.patch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
From 92da7c2cfeeea36d651142f47e570dd5076bc166 Mon Sep 17 00:00:00 2001
From: Wilco Dijkstra <wilco.dijkstra@arm.com>
Date: Thu, 21 Mar 2024 16:48:33 +0000
Subject: [PATCH] AArch64: Check kernel version for SVE ifuncs
Old Linux kernels disable SVE after every system call. Calling the
SVE-optimized memcpy afterwards will then cause a trap to reenable SVE.
As a result, applications with a high use of syscalls may run slower with
the SVE memcpy. This is true for kernels between 4.15.0 and before 6.2.0,
except for 5.14.0 which was patched. Avoid this by checking the kernel
version and selecting the SVE ifunc on modern kernels.
Parse the kernel version reported by uname() into a 24-bit kernel.major.minor
value without calling any library functions. If uname() is not supported or
if the version format is not recognized, assume the kernel is modern.
Tested-by: Florian Weimer <fweimer@redhat.com>
Reviewed-by: Szabolcs Nagy <szabolcs.nagy@arm.com>
(cherry picked from commit 2e94e2f5d2bf2de124c8ad7da85463355e54ccb2)
---
sysdeps/aarch64/multiarch/init-arch.h | 2 +
sysdeps/aarch64/multiarch/memcpy.c | 2 +-
sysdeps/aarch64/multiarch/memmove.c | 2 +-
.../unix/sysv/linux/aarch64/cpu-features.c | 48 +++++++++++++++++++
.../unix/sysv/linux/aarch64/cpu-features.h | 1 +
5 files changed, 53 insertions(+), 2 deletions(-)
diff --git a/sysdeps/aarch64/multiarch/init-arch.h b/sysdeps/aarch64/multiarch/init-arch.h
index e23e6ff290..daef631e04 100644
--- a/sysdeps/aarch64/multiarch/init-arch.h
+++ b/sysdeps/aarch64/multiarch/init-arch.h
@@ -36,5 +36,7 @@
MTE_ENABLED (); \
bool __attribute__((unused)) sve = \
GLRO(dl_aarch64_cpu_features).sve; \
+ bool __attribute__((unused)) prefer_sve_ifuncs = \
+ GLRO(dl_aarch64_cpu_features).prefer_sve_ifuncs; \
bool __attribute__((unused)) mops = \
GLRO(dl_aarch64_cpu_features).mops;
diff --git a/sysdeps/aarch64/multiarch/memcpy.c b/sysdeps/aarch64/multiarch/memcpy.c
index 6471fe82e3..e7c7795db6 100644
--- a/sysdeps/aarch64/multiarch/memcpy.c
+++ b/sysdeps/aarch64/multiarch/memcpy.c
@@ -47,7 +47,7 @@ select_memcpy_ifunc (void)
{
if (IS_A64FX (midr))
return __memcpy_a64fx;
- return __memcpy_sve;
+ return prefer_sve_ifuncs ? __memcpy_sve : __memcpy_generic;
}
if (IS_THUNDERX (midr))
diff --git a/sysdeps/aarch64/multiarch/memmove.c b/sysdeps/aarch64/multiarch/memmove.c
index 7602a5d57d..6b77166851 100644
--- a/sysdeps/aarch64/multiarch/memmove.c
+++ b/sysdeps/aarch64/multiarch/memmove.c
@@ -47,7 +47,7 @@ select_memmove_ifunc (void)
{
if (IS_A64FX (midr))
return __memmove_a64fx;
- return __memmove_sve;
+ return prefer_sve_ifuncs ? __memmove_sve : __memmove_generic;
}
if (IS_THUNDERX (midr))
diff --git a/sysdeps/unix/sysv/linux/aarch64/cpu-features.c b/sysdeps/unix/sysv/linux/aarch64/cpu-features.c
index a11a86efab..4a205a6b35 100644
--- a/sysdeps/unix/sysv/linux/aarch64/cpu-features.c
+++ b/sysdeps/unix/sysv/linux/aarch64/cpu-features.c
@@ -20,6 +20,7 @@
#include <sys/auxv.h>
#include <elf/dl-hwcaps.h>
#include <sys/prctl.h>
+#include <sys/utsname.h>
#define DCZID_DZP_MASK (1 << 4)
#define DCZID_BS_MASK (0xf)
@@ -57,6 +58,46 @@ get_midr_from_mcpu (const char *mcpu)
return UINT64_MAX;
}
+#if __LINUX_KERNEL_VERSION < 0x060200
+
+/* Return true if we prefer using SVE in string ifuncs. Old kernels disable
+ SVE after every system call which results in unnecessary traps if memcpy
+ uses SVE. This is true for kernels between 4.15.0 and before 6.2.0, except
+ for 5.14.0 which was patched. For these versions return false to avoid using
+ SVE ifuncs.
+ Parse the kernel version into a 24-bit kernel.major.minor value without
+ calling any library functions. If uname() is not supported or if the version
+ format is not recognized, assume the kernel is modern and return true. */
+
+static inline bool
+prefer_sve_ifuncs (void)
+{
+ struct utsname buf;
+ const char *p = &buf.release[0];
+ int kernel = 0;
+ int val;
+
+ if (__uname (&buf) < 0)
+ return true;
+
+ for (int shift = 16; shift >= 0; shift -= 8)
+ {
+ for (val = 0; *p >= '0' && *p <= '9'; p++)
+ val = val * 10 + *p - '0';
+ kernel |= (val & 255) << shift;
+ if (*p++ != '.')
+ break;
+ }
+
+ if (kernel >= 0x060200 || kernel == 0x050e00)
+ return true;
+ if (kernel >= 0x040f00)
+ return false;
+ return true;
+}
+
+#endif
+
static inline void
init_cpu_features (struct cpu_features *cpu_features)
{
@@ -119,6 +160,13 @@ init_cpu_features (struct cpu_features *cpu_features)
/* Check if SVE is supported. */
cpu_features->sve = GLRO (dl_hwcap) & HWCAP_SVE;
+ cpu_features->prefer_sve_ifuncs = cpu_features->sve;
+
+#if __LINUX_KERNEL_VERSION < 0x060200
+ if (cpu_features->sve)
+ cpu_features->prefer_sve_ifuncs = prefer_sve_ifuncs ();
+#endif
+
/* Check if MOPS is supported. */
cpu_features->mops = GLRO (dl_hwcap2) & HWCAP2_MOPS;
}
diff --git a/sysdeps/unix/sysv/linux/aarch64/cpu-features.h b/sysdeps/unix/sysv/linux/aarch64/cpu-features.h
index 2cf745cd19..351a619dcb 100644
--- a/sysdeps/unix/sysv/linux/aarch64/cpu-features.h
+++ b/sysdeps/unix/sysv/linux/aarch64/cpu-features.h
@@ -71,6 +71,7 @@ struct cpu_features
/* Currently, the GLIBC memory tagging tunable only defines 8 bits. */
uint8_t mte_state;
bool sve;
+ bool prefer_sve_ifuncs;
bool mops;
};
--
2.44.0