1 /* SPDX-License-Identifier: GPL-2.0 */
3 * Author: Lu Zeng <zenglu@loongson.cn>
4 * Pei Huang <huangpei@loongson.cn>
5 * Huacai Chen <chenhuacai@loongson.cn>
7 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
9 #include <linux/export.h>
11 #include <asm/asmmacro.h>
12 #include <asm/asm-extable.h>
13 #include <asm/asm-offsets.h>
14 #include <asm/errno.h>
15 #include <asm/fpregdef.h>
16 #include <asm/loongarch.h>
17 #include <asm/regdef.h>
19 #define FPU_REG_WIDTH 8
20 #define LSX_REG_WIDTH 16
21 #define LASX_REG_WIDTH 32
23 .macro EX insn, reg, src, offs
24 .ex\@: \insn \reg, \src, \offs
25 _asm_extable .ex\@, fault
28 .macro sc_save_fp base
29 EX fst.d $f0, \base, (0 * FPU_REG_WIDTH)
30 EX fst.d $f1, \base, (1 * FPU_REG_WIDTH)
31 EX fst.d $f2, \base, (2 * FPU_REG_WIDTH)
32 EX fst.d $f3, \base, (3 * FPU_REG_WIDTH)
33 EX fst.d $f4, \base, (4 * FPU_REG_WIDTH)
34 EX fst.d $f5, \base, (5 * FPU_REG_WIDTH)
35 EX fst.d $f6, \base, (6 * FPU_REG_WIDTH)
36 EX fst.d $f7, \base, (7 * FPU_REG_WIDTH)
37 EX fst.d $f8, \base, (8 * FPU_REG_WIDTH)
38 EX fst.d $f9, \base, (9 * FPU_REG_WIDTH)
39 EX fst.d $f10, \base, (10 * FPU_REG_WIDTH)
40 EX fst.d $f11, \base, (11 * FPU_REG_WIDTH)
41 EX fst.d $f12, \base, (12 * FPU_REG_WIDTH)
42 EX fst.d $f13, \base, (13 * FPU_REG_WIDTH)
43 EX fst.d $f14, \base, (14 * FPU_REG_WIDTH)
44 EX fst.d $f15, \base, (15 * FPU_REG_WIDTH)
45 EX fst.d $f16, \base, (16 * FPU_REG_WIDTH)
46 EX fst.d $f17, \base, (17 * FPU_REG_WIDTH)
47 EX fst.d $f18, \base, (18 * FPU_REG_WIDTH)
48 EX fst.d $f19, \base, (19 * FPU_REG_WIDTH)
49 EX fst.d $f20, \base, (20 * FPU_REG_WIDTH)
50 EX fst.d $f21, \base, (21 * FPU_REG_WIDTH)
51 EX fst.d $f22, \base, (22 * FPU_REG_WIDTH)
52 EX fst.d $f23, \base, (23 * FPU_REG_WIDTH)
53 EX fst.d $f24, \base, (24 * FPU_REG_WIDTH)
54 EX fst.d $f25, \base, (25 * FPU_REG_WIDTH)
55 EX fst.d $f26, \base, (26 * FPU_REG_WIDTH)
56 EX fst.d $f27, \base, (27 * FPU_REG_WIDTH)
57 EX fst.d $f28, \base, (28 * FPU_REG_WIDTH)
58 EX fst.d $f29, \base, (29 * FPU_REG_WIDTH)
59 EX fst.d $f30, \base, (30 * FPU_REG_WIDTH)
60 EX fst.d $f31, \base, (31 * FPU_REG_WIDTH)
63 .macro sc_restore_fp base
64 EX fld.d $f0, \base, (0 * FPU_REG_WIDTH)
65 EX fld.d $f1, \base, (1 * FPU_REG_WIDTH)
66 EX fld.d $f2, \base, (2 * FPU_REG_WIDTH)
67 EX fld.d $f3, \base, (3 * FPU_REG_WIDTH)
68 EX fld.d $f4, \base, (4 * FPU_REG_WIDTH)
69 EX fld.d $f5, \base, (5 * FPU_REG_WIDTH)
70 EX fld.d $f6, \base, (6 * FPU_REG_WIDTH)
71 EX fld.d $f7, \base, (7 * FPU_REG_WIDTH)
72 EX fld.d $f8, \base, (8 * FPU_REG_WIDTH)
73 EX fld.d $f9, \base, (9 * FPU_REG_WIDTH)
74 EX fld.d $f10, \base, (10 * FPU_REG_WIDTH)
75 EX fld.d $f11, \base, (11 * FPU_REG_WIDTH)
76 EX fld.d $f12, \base, (12 * FPU_REG_WIDTH)
77 EX fld.d $f13, \base, (13 * FPU_REG_WIDTH)
78 EX fld.d $f14, \base, (14 * FPU_REG_WIDTH)
79 EX fld.d $f15, \base, (15 * FPU_REG_WIDTH)
80 EX fld.d $f16, \base, (16 * FPU_REG_WIDTH)
81 EX fld.d $f17, \base, (17 * FPU_REG_WIDTH)
82 EX fld.d $f18, \base, (18 * FPU_REG_WIDTH)
83 EX fld.d $f19, \base, (19 * FPU_REG_WIDTH)
84 EX fld.d $f20, \base, (20 * FPU_REG_WIDTH)
85 EX fld.d $f21, \base, (21 * FPU_REG_WIDTH)
86 EX fld.d $f22, \base, (22 * FPU_REG_WIDTH)
87 EX fld.d $f23, \base, (23 * FPU_REG_WIDTH)
88 EX fld.d $f24, \base, (24 * FPU_REG_WIDTH)
89 EX fld.d $f25, \base, (25 * FPU_REG_WIDTH)
90 EX fld.d $f26, \base, (26 * FPU_REG_WIDTH)
91 EX fld.d $f27, \base, (27 * FPU_REG_WIDTH)
92 EX fld.d $f28, \base, (28 * FPU_REG_WIDTH)
93 EX fld.d $f29, \base, (29 * FPU_REG_WIDTH)
94 EX fld.d $f30, \base, (30 * FPU_REG_WIDTH)
95 EX fld.d $f31, \base, (31 * FPU_REG_WIDTH)
98 .macro sc_save_fcc base, tmp0, tmp1
101 movcf2gr \tmp0, $fcc1
102 bstrins.d \tmp1, \tmp0, 15, 8
103 movcf2gr \tmp0, $fcc2
104 bstrins.d \tmp1, \tmp0, 23, 16
105 movcf2gr \tmp0, $fcc3
106 bstrins.d \tmp1, \tmp0, 31, 24
107 movcf2gr \tmp0, $fcc4
108 bstrins.d \tmp1, \tmp0, 39, 32
109 movcf2gr \tmp0, $fcc5
110 bstrins.d \tmp1, \tmp0, 47, 40
111 movcf2gr \tmp0, $fcc6
112 bstrins.d \tmp1, \tmp0, 55, 48
113 movcf2gr \tmp0, $fcc7
114 bstrins.d \tmp1, \tmp0, 63, 56
115 EX st.d \tmp1, \base, 0
118 .macro sc_restore_fcc base, tmp0, tmp1
119 EX ld.d \tmp0, \base, 0
120 bstrpick.d \tmp1, \tmp0, 7, 0
121 movgr2cf $fcc0, \tmp1
122 bstrpick.d \tmp1, \tmp0, 15, 8
123 movgr2cf $fcc1, \tmp1
124 bstrpick.d \tmp1, \tmp0, 23, 16
125 movgr2cf $fcc2, \tmp1
126 bstrpick.d \tmp1, \tmp0, 31, 24
127 movgr2cf $fcc3, \tmp1
128 bstrpick.d \tmp1, \tmp0, 39, 32
129 movgr2cf $fcc4, \tmp1
130 bstrpick.d \tmp1, \tmp0, 47, 40
131 movgr2cf $fcc5, \tmp1
132 bstrpick.d \tmp1, \tmp0, 55, 48
133 movgr2cf $fcc6, \tmp1
134 bstrpick.d \tmp1, \tmp0, 63, 56
135 movgr2cf $fcc7, \tmp1
138 .macro sc_save_fcsr base, tmp0
139 movfcsr2gr \tmp0, fcsr0
140 EX st.w \tmp0, \base, 0
143 .macro sc_restore_fcsr base, tmp0
144 EX ld.w \tmp0, \base, 0
145 movgr2fcsr fcsr0, \tmp0
148 .macro sc_save_lsx base
149 #ifdef CONFIG_CPU_HAS_LSX
150 EX vst $vr0, \base, (0 * LSX_REG_WIDTH)
151 EX vst $vr1, \base, (1 * LSX_REG_WIDTH)
152 EX vst $vr2, \base, (2 * LSX_REG_WIDTH)
153 EX vst $vr3, \base, (3 * LSX_REG_WIDTH)
154 EX vst $vr4, \base, (4 * LSX_REG_WIDTH)
155 EX vst $vr5, \base, (5 * LSX_REG_WIDTH)
156 EX vst $vr6, \base, (6 * LSX_REG_WIDTH)
157 EX vst $vr7, \base, (7 * LSX_REG_WIDTH)
158 EX vst $vr8, \base, (8 * LSX_REG_WIDTH)
159 EX vst $vr9, \base, (9 * LSX_REG_WIDTH)
160 EX vst $vr10, \base, (10 * LSX_REG_WIDTH)
161 EX vst $vr11, \base, (11 * LSX_REG_WIDTH)
162 EX vst $vr12, \base, (12 * LSX_REG_WIDTH)
163 EX vst $vr13, \base, (13 * LSX_REG_WIDTH)
164 EX vst $vr14, \base, (14 * LSX_REG_WIDTH)
165 EX vst $vr15, \base, (15 * LSX_REG_WIDTH)
166 EX vst $vr16, \base, (16 * LSX_REG_WIDTH)
167 EX vst $vr17, \base, (17 * LSX_REG_WIDTH)
168 EX vst $vr18, \base, (18 * LSX_REG_WIDTH)
169 EX vst $vr19, \base, (19 * LSX_REG_WIDTH)
170 EX vst $vr20, \base, (20 * LSX_REG_WIDTH)
171 EX vst $vr21, \base, (21 * LSX_REG_WIDTH)
172 EX vst $vr22, \base, (22 * LSX_REG_WIDTH)
173 EX vst $vr23, \base, (23 * LSX_REG_WIDTH)
174 EX vst $vr24, \base, (24 * LSX_REG_WIDTH)
175 EX vst $vr25, \base, (25 * LSX_REG_WIDTH)
176 EX vst $vr26, \base, (26 * LSX_REG_WIDTH)
177 EX vst $vr27, \base, (27 * LSX_REG_WIDTH)
178 EX vst $vr28, \base, (28 * LSX_REG_WIDTH)
179 EX vst $vr29, \base, (29 * LSX_REG_WIDTH)
180 EX vst $vr30, \base, (30 * LSX_REG_WIDTH)
181 EX vst $vr31, \base, (31 * LSX_REG_WIDTH)
185 .macro sc_restore_lsx base
186 #ifdef CONFIG_CPU_HAS_LSX
187 EX vld $vr0, \base, (0 * LSX_REG_WIDTH)
188 EX vld $vr1, \base, (1 * LSX_REG_WIDTH)
189 EX vld $vr2, \base, (2 * LSX_REG_WIDTH)
190 EX vld $vr3, \base, (3 * LSX_REG_WIDTH)
191 EX vld $vr4, \base, (4 * LSX_REG_WIDTH)
192 EX vld $vr5, \base, (5 * LSX_REG_WIDTH)
193 EX vld $vr6, \base, (6 * LSX_REG_WIDTH)
194 EX vld $vr7, \base, (7 * LSX_REG_WIDTH)
195 EX vld $vr8, \base, (8 * LSX_REG_WIDTH)
196 EX vld $vr9, \base, (9 * LSX_REG_WIDTH)
197 EX vld $vr10, \base, (10 * LSX_REG_WIDTH)
198 EX vld $vr11, \base, (11 * LSX_REG_WIDTH)
199 EX vld $vr12, \base, (12 * LSX_REG_WIDTH)
200 EX vld $vr13, \base, (13 * LSX_REG_WIDTH)
201 EX vld $vr14, \base, (14 * LSX_REG_WIDTH)
202 EX vld $vr15, \base, (15 * LSX_REG_WIDTH)
203 EX vld $vr16, \base, (16 * LSX_REG_WIDTH)
204 EX vld $vr17, \base, (17 * LSX_REG_WIDTH)
205 EX vld $vr18, \base, (18 * LSX_REG_WIDTH)
206 EX vld $vr19, \base, (19 * LSX_REG_WIDTH)
207 EX vld $vr20, \base, (20 * LSX_REG_WIDTH)
208 EX vld $vr21, \base, (21 * LSX_REG_WIDTH)
209 EX vld $vr22, \base, (22 * LSX_REG_WIDTH)
210 EX vld $vr23, \base, (23 * LSX_REG_WIDTH)
211 EX vld $vr24, \base, (24 * LSX_REG_WIDTH)
212 EX vld $vr25, \base, (25 * LSX_REG_WIDTH)
213 EX vld $vr26, \base, (26 * LSX_REG_WIDTH)
214 EX vld $vr27, \base, (27 * LSX_REG_WIDTH)
215 EX vld $vr28, \base, (28 * LSX_REG_WIDTH)
216 EX vld $vr29, \base, (29 * LSX_REG_WIDTH)
217 EX vld $vr30, \base, (30 * LSX_REG_WIDTH)
218 EX vld $vr31, \base, (31 * LSX_REG_WIDTH)
222 .macro sc_save_lasx base
223 #ifdef CONFIG_CPU_HAS_LASX
224 EX xvst $xr0, \base, (0 * LASX_REG_WIDTH)
225 EX xvst $xr1, \base, (1 * LASX_REG_WIDTH)
226 EX xvst $xr2, \base, (2 * LASX_REG_WIDTH)
227 EX xvst $xr3, \base, (3 * LASX_REG_WIDTH)
228 EX xvst $xr4, \base, (4 * LASX_REG_WIDTH)
229 EX xvst $xr5, \base, (5 * LASX_REG_WIDTH)
230 EX xvst $xr6, \base, (6 * LASX_REG_WIDTH)
231 EX xvst $xr7, \base, (7 * LASX_REG_WIDTH)
232 EX xvst $xr8, \base, (8 * LASX_REG_WIDTH)
233 EX xvst $xr9, \base, (9 * LASX_REG_WIDTH)
234 EX xvst $xr10, \base, (10 * LASX_REG_WIDTH)
235 EX xvst $xr11, \base, (11 * LASX_REG_WIDTH)
236 EX xvst $xr12, \base, (12 * LASX_REG_WIDTH)
237 EX xvst $xr13, \base, (13 * LASX_REG_WIDTH)
238 EX xvst $xr14, \base, (14 * LASX_REG_WIDTH)
239 EX xvst $xr15, \base, (15 * LASX_REG_WIDTH)
240 EX xvst $xr16, \base, (16 * LASX_REG_WIDTH)
241 EX xvst $xr17, \base, (17 * LASX_REG_WIDTH)
242 EX xvst $xr18, \base, (18 * LASX_REG_WIDTH)
243 EX xvst $xr19, \base, (19 * LASX_REG_WIDTH)
244 EX xvst $xr20, \base, (20 * LASX_REG_WIDTH)
245 EX xvst $xr21, \base, (21 * LASX_REG_WIDTH)
246 EX xvst $xr22, \base, (22 * LASX_REG_WIDTH)
247 EX xvst $xr23, \base, (23 * LASX_REG_WIDTH)
248 EX xvst $xr24, \base, (24 * LASX_REG_WIDTH)
249 EX xvst $xr25, \base, (25 * LASX_REG_WIDTH)
250 EX xvst $xr26, \base, (26 * LASX_REG_WIDTH)
251 EX xvst $xr27, \base, (27 * LASX_REG_WIDTH)
252 EX xvst $xr28, \base, (28 * LASX_REG_WIDTH)
253 EX xvst $xr29, \base, (29 * LASX_REG_WIDTH)
254 EX xvst $xr30, \base, (30 * LASX_REG_WIDTH)
255 EX xvst $xr31, \base, (31 * LASX_REG_WIDTH)
259 .macro sc_restore_lasx base
260 #ifdef CONFIG_CPU_HAS_LASX
261 EX xvld $xr0, \base, (0 * LASX_REG_WIDTH)
262 EX xvld $xr1, \base, (1 * LASX_REG_WIDTH)
263 EX xvld $xr2, \base, (2 * LASX_REG_WIDTH)
264 EX xvld $xr3, \base, (3 * LASX_REG_WIDTH)
265 EX xvld $xr4, \base, (4 * LASX_REG_WIDTH)
266 EX xvld $xr5, \base, (5 * LASX_REG_WIDTH)
267 EX xvld $xr6, \base, (6 * LASX_REG_WIDTH)
268 EX xvld $xr7, \base, (7 * LASX_REG_WIDTH)
269 EX xvld $xr8, \base, (8 * LASX_REG_WIDTH)
270 EX xvld $xr9, \base, (9 * LASX_REG_WIDTH)
271 EX xvld $xr10, \base, (10 * LASX_REG_WIDTH)
272 EX xvld $xr11, \base, (11 * LASX_REG_WIDTH)
273 EX xvld $xr12, \base, (12 * LASX_REG_WIDTH)
274 EX xvld $xr13, \base, (13 * LASX_REG_WIDTH)
275 EX xvld $xr14, \base, (14 * LASX_REG_WIDTH)
276 EX xvld $xr15, \base, (15 * LASX_REG_WIDTH)
277 EX xvld $xr16, \base, (16 * LASX_REG_WIDTH)
278 EX xvld $xr17, \base, (17 * LASX_REG_WIDTH)
279 EX xvld $xr18, \base, (18 * LASX_REG_WIDTH)
280 EX xvld $xr19, \base, (19 * LASX_REG_WIDTH)
281 EX xvld $xr20, \base, (20 * LASX_REG_WIDTH)
282 EX xvld $xr21, \base, (21 * LASX_REG_WIDTH)
283 EX xvld $xr22, \base, (22 * LASX_REG_WIDTH)
284 EX xvld $xr23, \base, (23 * LASX_REG_WIDTH)
285 EX xvld $xr24, \base, (24 * LASX_REG_WIDTH)
286 EX xvld $xr25, \base, (25 * LASX_REG_WIDTH)
287 EX xvld $xr26, \base, (26 * LASX_REG_WIDTH)
288 EX xvld $xr27, \base, (27 * LASX_REG_WIDTH)
289 EX xvld $xr28, \base, (28 * LASX_REG_WIDTH)
290 EX xvld $xr29, \base, (29 * LASX_REG_WIDTH)
291 EX xvld $xr30, \base, (30 * LASX_REG_WIDTH)
292 EX xvld $xr31, \base, (31 * LASX_REG_WIDTH)
297 * Save a thread's fp context.
299 SYM_FUNC_START(_save_fp)
301 fpu_save_double a0 t1 # clobbers t1
302 fpu_save_cc a0 t1 t2 # clobbers t1, t2
304 SYM_FUNC_END(_save_fp)
305 EXPORT_SYMBOL(_save_fp)
308 * Restore a thread's fp context.
310 SYM_FUNC_START(_restore_fp)
311 fpu_restore_double a0 t1 # clobbers t1
312 fpu_restore_csr a0 t1
313 fpu_restore_cc a0 t1 t2 # clobbers t1, t2
315 SYM_FUNC_END(_restore_fp)
317 #ifdef CONFIG_CPU_HAS_LSX
320 * Save a thread's LSX vector context.
322 SYM_FUNC_START(_save_lsx)
323 lsx_save_all a0 t1 t2
325 SYM_FUNC_END(_save_lsx)
326 EXPORT_SYMBOL(_save_lsx)
329 * Restore a thread's LSX vector context.
331 SYM_FUNC_START(_restore_lsx)
332 lsx_restore_all a0 t1 t2
334 SYM_FUNC_END(_restore_lsx)
336 SYM_FUNC_START(_save_lsx_upper)
337 lsx_save_all_upper a0 t0 t1
339 SYM_FUNC_END(_save_lsx_upper)
341 SYM_FUNC_START(_restore_lsx_upper)
342 lsx_restore_all_upper a0 t0 t1
344 SYM_FUNC_END(_restore_lsx_upper)
346 SYM_FUNC_START(_init_lsx_upper)
347 lsx_init_all_upper t1
349 SYM_FUNC_END(_init_lsx_upper)
352 #ifdef CONFIG_CPU_HAS_LASX
355 * Save a thread's LASX vector context.
357 SYM_FUNC_START(_save_lasx)
358 lasx_save_all a0 t1 t2
360 SYM_FUNC_END(_save_lasx)
361 EXPORT_SYMBOL(_save_lasx)
364 * Restore a thread's LASX vector context.
366 SYM_FUNC_START(_restore_lasx)
367 lasx_restore_all a0 t1 t2
369 SYM_FUNC_END(_restore_lasx)
371 SYM_FUNC_START(_save_lasx_upper)
372 lasx_save_all_upper a0 t0 t1
374 SYM_FUNC_END(_save_lasx_upper)
376 SYM_FUNC_START(_restore_lasx_upper)
377 lasx_restore_all_upper a0 t0 t1
379 SYM_FUNC_END(_restore_lasx_upper)
381 SYM_FUNC_START(_init_lasx_upper)
382 lasx_init_all_upper t1
384 SYM_FUNC_END(_init_lasx_upper)
388 * Load the FPU with signalling NANS. This bit pattern we're using has
389 * the property that no matter whether considered as single or as double
390 * precision represents signaling NANS.
392 * The value to initialize fcsr0 to comes in $a0.
395 SYM_FUNC_START(_init_fpu)
396 li.w t1, CSR_EUEN_FPEN
397 csrxchg t1, t1, LOONGARCH_CSR_EUEN
437 SYM_FUNC_END(_init_fpu)
444 SYM_FUNC_START(_save_fp_context)
450 SYM_FUNC_END(_save_fp_context)
457 SYM_FUNC_START(_restore_fp_context)
459 sc_restore_fcc a1 t1 t2
460 sc_restore_fcsr a2 t1
463 SYM_FUNC_END(_restore_fp_context)
470 SYM_FUNC_START(_save_lsx_context)
471 sc_save_fcc a1, t0, t1
476 SYM_FUNC_END(_save_lsx_context)
483 SYM_FUNC_START(_restore_lsx_context)
485 sc_restore_fcc a1, t1, t2
486 sc_restore_fcsr a2, t1
489 SYM_FUNC_END(_restore_lsx_context)
496 SYM_FUNC_START(_save_lasx_context)
497 sc_save_fcc a1, t0, t1
502 SYM_FUNC_END(_save_lasx_context)
509 SYM_FUNC_START(_restore_lasx_context)
511 sc_restore_fcc a1, t1, t2
512 sc_restore_fcsr a2, t1
515 SYM_FUNC_END(_restore_lasx_context)
517 SYM_FUNC_START(fault)
518 li.w a0, -EFAULT # failure