1 // Copyright 2026 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build !purego
6
7 // This file contains constant-time, 64-bit assembly implementation of
8 // P256. The optimizations performed here are described in detail in:
9 // S.Gueron and V.Krasnov, "Fast prime field elliptic-curve cryptography with
10 // 256-bit primes"
11 // http://link.springer.com/article/10.1007%2Fs13389-014-0090-x
12 // https://eprint.iacr.org/2013/816.pdf
13
14 #include "textflag.h"
15
16 #define res_ptr R4
17 #define a_ptr R5
18 #define b_ptr R6
19
20 #define acc0 R7
21 #define acc1 R8
22 #define acc2 R9
23 #define acc3 R10
24 #define acc4 R11
25 #define acc5 R12
26 #define acc6 R13
27 #define acc7 R14
28
29 #define t0 R15
30 #define t1 R16
31 #define t2 R17
32 #define t3 R18
33 #define t4 R19
34 #define t5 R24
35 #define t6 R25
36
37 #define x0 R26
38 #define x1 R27
39 #define x2 R28
40 #define x3 R20
41
42 #define y0 R23
43 #define y1 a_ptr
44 #define y2 R21
45 #define y3 R31
46
47 #define hlp0 b_ptr
48 #define hlp1 t2
49
50 DATA p256const0<>+0x00(SB)/8, $0x00000000ffffffff
51 DATA p256const1<>+0x00(SB)/8, $0xffffffff00000001
52 DATA p256ordK0<>+0x00(SB)/8, $0xccd1c8aaee00bc4f
53 DATA p256ord<>+0x00(SB)/8, $0xf3b9cac2fc632551
54 DATA p256ord<>+0x08(SB)/8, $0xbce6faada7179e84
55 DATA p256ord<>+0x10(SB)/8, $0xffffffffffffffff
56 DATA p256ord<>+0x18(SB)/8, $0xffffffff00000000
57 DATA p256one<>+0x00(SB)/8, $0x0000000000000001
58 DATA p256one<>+0x08(SB)/8, $0xffffffff00000000
59 DATA p256one<>+0x10(SB)/8, $0xffffffffffffffff
60 DATA p256one<>+0x18(SB)/8, $0x00000000fffffffe
61 GLOBL p256const0<>(SB), (NOPTR|RODATA), $8
62 GLOBL p256const1<>(SB), (NOPTR|RODATA), $8
63 GLOBL p256ordK0<>(SB), (NOPTR|RODATA), $8
64 GLOBL p256ord<>(SB), (NOPTR|RODATA), $32
65 GLOBL p256one<>(SB), (NOPTR|RODATA), $32
66
67 // func p256MovCond(res, a, b *P256Point, cond int)
68 // If cond is 0, sets res = b, otherwise sets res = a.
69 TEXT ·p256MovCond(SB),NOSPLIT,$0
70 MOVV res+0(FP), res_ptr
71 MOVV a+8(FP), a_ptr
72 MOVV b+16(FP), b_ptr
73 MOVV cond+24(FP), t0
74
75 // mask = -(cond != 0)
76 SGTU t0, R0, t1
77 SUBV t1, R0, t1
78
79 // Select each limb as:
80 // b ^ ((a ^ b) & mask)
81 //
82 // The loop count is fixed at 12 and does not depend on cond.
83 MOVV $0, t2
84 MOVV $12, x0
85 MOVV $0, t3
86
87 p256MovCond_loop:
88 MOVV (t3)(a_ptr), t4
89 MOVV (t3)(b_ptr), t5
90 XOR t4, t5, t6
91 AND t1, t6, t6
92 XOR t5, t6, t6
93 MOVV t6, (t3)(res_ptr)
94
95 ADDV $8, t3, t3
96 ADDV $1, t2, t2
97 BNE t2, x0, p256MovCond_loop
98 RET
99
100 // func p256NegCond(val *p256Element, cond int)
101 // If cond is not 0, sets val = -val mod p (aka: val = P - val).
102 TEXT ·p256NegCond(SB),NOSPLIT,$0
103 MOVV val+0(FP), res_ptr
104 MOVV cond+8(FP), t0
105
106 // mask = -(cond != 0)
107 SGTU t0, R0, acc4
108 SUBV acc4, R0, acc4
109
110 // acc = poly
111 MOVV $-1, acc0
112 MOVV p256const0<>(SB), acc1
113 MOVV $0, acc2
114 MOVV p256const1<>(SB), acc3
115
116 // Load the original value
117 MOVV (0*8)(res_ptr), t0
118 MOVV (1*8)(res_ptr), t1
119 MOVV (2*8)(res_ptr), t2
120 MOVV (3*8)(res_ptr), t3
121
122 // Keep the original value for the final constant-time selection.
123 MOVV t0, x0
124 MOVV t1, x1
125 MOVV t2, x2
126 MOVV t3, x3
127
128 // Speculatively subtract
129 SUBV t0, acc0, t0
130 SUBV t1, acc1, t5 // x1 - y1 = z1', borrow if z1' > x1
131 SGTU t0, acc0, t4
132 SUBV t4, t5, t1 // z1' - c0 = z1, borrow if z1 > z1'
133 SGTU t5, acc1, t6
134 SGTU t1, t5, t4
135 OR t4, t6
136 SUBV t2, acc2, t5
137 SUBV t6, t5, t2
138 SGTU t5, acc2, t4
139 SGTU t2, t5, t5
140 OR t4, t5
141 SUBV t3, acc3, t3
142 SUBV t5, t3
143
144 // Select original value when cond == 0, negated value otherwise.
145 // result = original ^ ((original ^ negated) & mask)
146 XOR x0, t0, t4
147 AND acc4, t4, t4
148 XOR x0, t4, t0
149
150 XOR x1, t1, t4
151 AND acc4, t4, t4
152 XOR x1, t4, t1
153
154 XOR x2, t2, t4
155 AND acc4, t4, t4
156 XOR x2, t4, t2
157
158 XOR x3, t3, t4
159 AND acc4, t4, t4
160 XOR x3, t4, t3
161
162 MOVV t0, (0*8)(res_ptr)
163 MOVV t1, (1*8)(res_ptr)
164 MOVV t2, (2*8)(res_ptr)
165 MOVV t3, (3*8)(res_ptr)
166 RET
167
168 // p256Select sets res to the point at index idx in the table.
169 // idx must be in [0, 16]. It executes in constant time.
170 // idx == 0 returns the point at infinity (all zeroes).
171 // idx in [1, 16] selects table[idx-1].
172 //
173 // func p256Select(res *P256Point, table *p256Table, idx int)
174 TEXT ·p256Select(SB),NOSPLIT,$0
175 MOVV idx+16(FP), t0
176 MOVV table+8(FP), a_ptr
177 MOVV res+0(FP), res_ptr
178
179 // allones is used to invert the constant-time equality mask.
180 MOVV $-1, x2
181
182 // Process all 12 limbs. For every limb, scan all 16 entries.
183 // Both loop counts are fixed and independent of idx.
184 MOVV $0, x0
185 MOVV $12, x1 // limb count
186 MOVV $17, x3 // entry bound: i runs 1..16 (16 entries)
187
188 p256Select_limb:
189 MOVV $1, t1
190 MOVV a_ptr, t5
191 MOVV $0, t6
192
193 p256Select_entry:
194 // i is 1..16. Build:
195 // mask = 0xffff...ffff iff idx == i
196 XOR t0, t1, t2
197 SUBV t2, R0, t3
198 OR t2, t3, t3
199 SRAV $63, t3, t3
200 XOR t3, x2, t3
201
202 MOVV (t5), t4
203 AND t3, t4, t4
204 OR t6, t4, t6
205
206 ADDV $96, t5, t5
207 ADDV $1, t1, t1
208 BNE t1, x3, p256Select_entry
209
210 MOVV t6, (res_ptr)
211 ADDV $8, a_ptr, a_ptr
212 ADDV $8, res_ptr, res_ptr
213 ADDV $1, x0, x0
214 BNE x0, x1, p256Select_limb
215
216 RET
217
218 // p256SelectAffine sets res to the point at index idx in the table.
219 // idx must be in [0, 32]. It executes in constant time.
220 // idx == 0 returns the point at infinity (all zeroes).
221 // idx in [1, 32] selects table[idx-1].
222 //
223 // func p256SelectAffine(res *p256AffinePoint, table *p256AffineTable, idx int)
224 TEXT ·p256SelectAffine(SB),NOSPLIT,$0
225 MOVV idx+16(FP), t0
226 MOVV table+8(FP), a_ptr
227 MOVV res+0(FP), res_ptr
228
229 MOVV $-1, x2
230
231 // Process all 8 limbs (x and y), scanning all 32 entries.
232 // The loop counts are fixed and independent of idx.
233 MOVV $0, x0
234 MOVV $8, x1
235 MOVV $33, x3 // entry bound: i runs 1..32 (32 entries)
236
237 p256SelectAffine_limb:
238 MOVV $1, t1
239 MOVV a_ptr, t5
240 MOVV $0, t6
241
242 p256SelectAffine_entry:
243 // i is 1..32. Build:
244 // mask = 0xffff...ffff iff idx == i
245 XOR t0, t1, t2
246 SUBV t2, R0, t3
247 OR t2, t3, t3
248 SRAV $63, t3, t3
249 XOR t3, x2, t3
250
251 MOVV (t5), t4
252 AND t3, t4, t4
253 OR t6, t4, t6
254
255 ADDV $64, t5, t5
256 ADDV $1, t1, t1
257 BNE t1, x3, p256SelectAffine_entry
258
259 MOVV t6, (res_ptr)
260 ADDV $8, a_ptr, a_ptr
261 ADDV $8, res_ptr, res_ptr
262 ADDV $1, x0, x0
263 BNE x0, x1, p256SelectAffine_limb
264
265 RET
266
267 // func p256FromMont(res, in *p256Element)
268 TEXT ·p256FromMont(SB),NOSPLIT,$0-16
269 MOVV res+0(FP), res_ptr
270 MOVV in+8(FP), a_ptr
271
272 // acc4/acc5 hold p256const0/p256const1 (acc4/acc5 are free here).
273 MOVV p256const0<>(SB), acc4
274 MOVV p256const1<>(SB), acc5
275
276 MOVV 0*8(a_ptr), acc0
277 MOVV 1*8(a_ptr), acc1
278 MOVV 2*8(a_ptr), acc2
279 MOVV 3*8(a_ptr), acc3
280
281 // Round 1: eliminate acc0
282 SLLV $32, acc0, t2 // sh = acc0<<32
283 SRLV $32, acc0, t0 // hi32 = acc0>>32
284 MULV acc0, acc5, t1 // lo = low(acc0*const1)
285 MULHVU acc0, acc5, acc0 // acc0 = high(acc0*const1)
286
287 ADDV t2, acc1, acc1 // acc1 += sh
288 SGTU t2, acc1, t3 // c1 = carry
289
290 ADDV t0, acc2, t4 // sum = acc2+hi32
291 SGTU t0, t4, t5
292 ADDV t4, t3, acc2 // acc2 = sum + c1
293 SGTU t4, acc2, t3
294 ADDV t5, t3, t3 // c2
295
296 ADDV t1, acc3, t4
297 SGTU t1, t4, t5
298 ADDV t4, t3, acc3 // acc3 = sum + c2
299 SGTU t4, acc3, t3
300 ADDV t5, t3, t3 // c3
301
302 ADDV t3, acc0, acc0 // acc0 += c3
303
304 // Round 2: eliminate acc1
305 SLLV $32, acc1, t2
306 SRLV $32, acc1, t0
307 MULV acc1, acc5, t1
308 MULHVU acc1, acc5, acc1
309
310 ADDV t2, acc2, acc2
311 SGTU t2, acc2, t3
312
313 ADDV t0, acc3, t4
314 SGTU t0, t4, t5
315 ADDV t4, t3, acc3
316 SGTU t4, acc3, t3
317 ADDV t5, t3, t3
318
319 ADDV t1, acc0, t4
320 SGTU t1, t4, t5
321 ADDV t4, t3, acc0
322 SGTU t4, acc0, t3
323 ADDV t5, t3, t3
324
325 ADDV t3, acc1, acc1
326
327 // Round 3: eliminate acc2
328 SLLV $32, acc2, t2
329 SRLV $32, acc2, t0
330 MULV acc2, acc5, t1
331 MULHVU acc2, acc5, acc2
332
333 ADDV t2, acc3, acc3
334 SGTU t2, acc3, t3
335
336 ADDV t0, acc0, t4
337 SGTU t0, t4, t5
338 ADDV t4, t3, acc0
339 SGTU t4, acc0, t3
340 ADDV t5, t3, t3
341
342 ADDV t1, acc1, t4
343 SGTU t1, t4, t5
344 ADDV t4, t3, acc1
345 SGTU t4, acc1, t3
346 ADDV t5, t3, t3
347
348 ADDV t3, acc2, acc2
349
350 // Round 4: eliminate acc3
351 SLLV $32, acc3, t2
352 SRLV $32, acc3, t0
353 MULV acc3, acc5, t1
354 MULHVU acc3, acc5, acc3
355
356 ADDV t2, acc0, acc0
357 SGTU t2, acc0, t3
358
359 ADDV t0, acc1, t4
360 SGTU t0, t4, t5
361 ADDV t4, t3, acc1
362 SGTU t4, acc1, t3
363 ADDV t5, t3, t3
364
365 ADDV t1, acc2, t4
366 SGTU t1, t4, t5
367 ADDV t4, t3, acc2
368 SGTU t4, acc2, t3
369 ADDV t5, t3, t3
370
371 ADDV t3, acc3, acc3
372
373 // Final conditional subtraction: if acc >= p, subtract p
374 // p256P = {0xFFFFFFFFFFFFFFFF, const0, 0, const1}
375 MOVV $-1, t2 // allones
376
377 SUBV t2, acc0, t0 // diff0 = acc0 - allones
378 SGTU t2, acc0, t1 // borrow0
379
380 SUBV acc4, acc1, x0 // tmp = acc1 - const0
381 SGTU acc4, acc1, x1
382 SUBV t1, x0, y1 // diff1 = tmp - borrow0
383 SGTU t1, x0, x2
384 ADDV x1, x2, t1 // borrow1
385
386 SUBV t1, acc2, x0 // diff2 = acc2 - 0 - borrow1
387 SGTU t1, acc2, t1 // borrow2
388
389 SUBV acc5, acc3, y2 // tmp = acc3 - const1
390 SGTU acc5, acc3, x3
391 SUBV t1, y2, y3 // diff3 = tmp - borrow2
392 SGTU t1, y2, y0
393 ADDV x3, y0, t1 // borrow3 (final borrow)
394
395 // mask = 0 - borrow3:
396 // borrow3 = 0 -> mask = 0 (select diff); borrow3 = 1 -> mask = all ones (keep acc).
397 SUBV t1, R0, t3 // t3 = mask
398 MOVV $-1, t4
399 XOR t3, t4, t4 // notmask = ^mask
400
401 AND t3, acc0, x1
402 AND t4, t0, x3
403 OR x1, x3, acc0
404
405 AND t3, acc1, x1
406 AND t4, y1, x3
407 OR x1, x3, acc1
408
409 AND t3, acc2, x1
410 AND t4, x0, x3
411 OR x1, x3, acc2
412
413 AND t3, acc3, x1
414 AND t4, y3, x3
415 OR x1, x3, acc3
416
417 MOVV acc0, 0*8(res_ptr)
418 MOVV acc1, 1*8(res_ptr)
419 MOVV acc2, 2*8(res_ptr)
420 MOVV acc3, 3*8(res_ptr)
421 RET
422
423 // input: x0-x3, y0-y3
424 // output: x0-x3 = (y0-y3) - (x0-x3) mod p256P
425 // uses: const0(p256const0<>), const1(p256const1<>)
426 // clobbers: x0-x3 (overwritten with the result), y0-y3, acc0-acc7,
427 // t0-t3
428 TEXT p256SubInternal<>(SB),NOSPLIT,$0
429 // diff = y - x; borrow chain accumulates in t0
430 SUBV x0, y0, acc0
431 SGTU x0, y0, t0 // t0 = borrow0 = (y0 < x0)
432
433 SUBV x1, y1, t1 // t1 = y1 - x1 (raw, before borrow0)
434 SGTU x1, y1, t2 // t2 = raw borrow
435 SUBV t0, t1, acc1 // acc1 = t1 - borrow0
436 SGTU t0, t1, t3 // t3 = borrow from subtracting borrow0
437 OR t2, t3, t0 // t0 = borrow1
438
439 SUBV x2, y2, t1
440 SGTU x2, y2, t2
441 SUBV t0, t1, acc2
442 SGTU t0, t1, t3
443 OR t2, t3, t0 // t0 = borrow2
444
445 SUBV x3, y3, t1
446 SGTU x3, y3, t2
447 SUBV t0, t1, acc3
448 SGTU t0, t1, t3
449 OR t2, t3, t0 // t0 = borrow3 (final borrow: 1 means y < x)
450
451 // acc4..acc7 = diff + p256P (candidate to select on borrow)
452 MOVV $-1, t1 // p256P limb0
453 MOVV p256const0<>(SB), t2 // p256P limb1
454 MOVV p256const1<>(SB), t3 // p256P limb3
455
456 ADDV t1, acc0, acc4
457 SGTU t1, acc4, x0 // x0 = carry0
458
459 ADDV t2, acc1, x1 // x1 = acc1 + const0 (raw)
460 SGTU t2, x1, x2 // x2 = raw carry
461 ADDV x0, x1, acc5
462 SGTU x0, acc5, x3 // x3 = carry from adding carry0
463 OR x2, x3, x0 // x0 = carry1
464
465 ADDV x0, acc2, acc6 // limb2 addend is 0; only carry in
466 SGTU x0, acc6, x1
467 MOVV x1, x0 // x0 = carry2
468
469 ADDV t3, acc3, x2 // x2 = acc3 + const1 (raw)
470 ADDV x0, x2, acc7 // acc7 = raw + carry2 (top carry discarded)
471
472 // constant-time select: borrow==0 -> diff, borrow==1 -> diff+P
473 SUBV t0, R0, t1 // t1 = mask: 0 (no borrow) or all-1s (borrow)
474 MOVV $-1, t2
475 XOR t1, t2, t2 // t2 = notmask
476
477 AND t2, acc0, t3
478 AND t1, acc4, y0
479 OR t3, y0, x0
480
481 AND t2, acc1, t3
482 AND t1, acc5, y0
483 OR t3, y0, x1
484
485 AND t2, acc2, t3
486 AND t1, acc6, y0
487 OR t3, y0, x2
488
489 AND t2, acc3, t3
490 AND t1, acc7, y0
491 OR t3, y0, x3
492
493 RET
494
495 // p256SqrInternal computes y0..y3 = x0..x3 * x0..x3 mod P (Montgomery domain).
496 // Result left in y0-y3, matching the arm64 output convention.
497 // Clobbers t0-t6, acc0-acc7, y0-y3, and hlp0 (b_ptr/R6) -- callers must
498 // not keep live state in b_ptr across the CALL.
499 TEXT p256SqrInternal<>(SB),NOSPLIT,$0
500 // x[1:] * x[0]
501 MULV x0, x1, acc1
502 MULHVU x0, x1, acc2
503
504 MULV x0, x2, t0
505 ADDV t0, acc2, acc2
506 SGTU t0, acc2, t1
507 MULHVU x0, x2, acc3
508 ADDV t1, acc3, acc3
509
510 MULV x0, x3, t0
511 ADDV t0, acc3, acc3
512 SGTU t0, acc3, t1
513 MULHVU x0, x3, acc4
514 ADDV t1, acc4, acc4
515
516 // x[2:] * x[1]
517 MULV x1, x2, t0
518 ADDV t0, acc3, acc3
519 SGTU t0, acc3, t2
520 MULHVU x1, x2, t1
521 ADDV t1, acc4, acc4
522 SGTU t1, acc4, t3
523 ADDV t2, acc4, acc4
524 SGTU t2, acc4, t2
525 OR t3, t2, t2
526 ADDV t2, R0, acc5 // acc5 = carry-out (0 or 1)
527
528 MULV x1, x3, t0
529 ADDV t0, acc4, acc4
530 SGTU t0, acc4, t1
531 MULHVU x1, x3, t3
532 ADDV t3, acc5, acc5
533 SGTU t3, acc5, t4
534 ADDV t1, acc5, acc5
535 SGTU t1, acc5, t5
536 OR t4, t5, hlp0
537
538 // x[3] * x[2]
539 MULV x2, x3, t0
540 ADDV t0, acc5, acc5
541 SGTU t0, acc5, t1
542 MULHVU x2, x3, acc6
543 ADDV t1, acc6, acc6
544 ADDV hlp0, acc6, acc6
545
546 MOVV $0, acc7
547
548 // double all cross-products (acc1..acc6)
549 SLLV $1, acc6, t0
550 SRLV $63, acc5, t1
551 OR t1, t0, t0
552 SLLV $1, acc5, t1
553 SRLV $63, acc4, t2
554 OR t2, t1, t1
555 SLLV $1, acc4, t2
556 SRLV $63, acc3, t3
557 OR t3, t2, t2
558 SLLV $1, acc3, t3
559 SRLV $63, acc2, t4
560 OR t4, t3, t3
561 SLLV $1, acc2, t4
562 SRLV $63, acc1, t5
563 OR t5, t4, t4
564 SLLV $1, acc1, t5
565 SRLV $63, acc6, acc7 // top carry-out of the double (from old acc6's high bit)
566 MOVV t5, acc1
567 MOVV t4, acc2
568 MOVV t3, acc3
569 MOVV t2, acc4
570 MOVV t1, acc5
571 MOVV t0, acc6
572
573 // diagonal products: acc0 = low(x0*x0); add high(x0^2), x1^2, x2^2, x3^2
574 MULV x0, x0, acc0
575 MULHVU x0, x0, t0
576 ADDV t0, acc1, acc1
577 SGTU t0, acc1, t1 // carry into acc2 (propagated below)
578
579 MULV x1, x1, t0
580 ADDV t0, acc2, acc2
581 SGTU t0, acc2, t2
582 ADDV t1, acc2, acc2
583 SGTU t1, acc2, t3
584 OR t2, t3, t1
585 MULHVU x1, x1, t0
586 ADDV t0, acc3, acc3
587 SGTU t0, acc3, t2
588 ADDV t1, acc3, acc3
589 SGTU t1, acc3, t3
590 OR t2, t3, t1
591
592 MULV x2, x2, t0
593 ADDV t0, acc4, acc4
594 SGTU t0, acc4, t2
595 ADDV t1, acc4, acc4
596 SGTU t1, acc4, t3
597 OR t2, t3, t1
598 MULHVU x2, x2, t0
599 ADDV t0, acc5, acc5
600 SGTU t0, acc5, t2
601 ADDV t1, acc5, acc5
602 SGTU t1, acc5, t3
603 OR t2, t3, t1
604
605 MULV x3, x3, t0
606 ADDV t0, acc6, acc6
607 SGTU t0, acc6, t2
608 ADDV t1, acc6, acc6
609 SGTU t1, acc6, t3
610 OR t2, t3, t1
611 MULHVU x3, x3, t0
612 ADDV t0, acc7, acc7
613 ADDV t1, acc7, acc7
614
615 // First reduction step
616 SLLV $32, acc0, t2
617 SRLV $32, acc0, t0
618 MOVV p256const1<>(SB), t6
619 MULV acc0, t6, t1
620 MULHVU acc0, t6, acc0
621 ADDV t2, acc1, acc1
622 SGTU t2, acc1, t3
623 ADDV t0, acc2, acc2
624 SGTU t0, acc2, t4
625 ADDV t3, acc2, acc2
626 SGTU t3, acc2, t5
627 OR t4, t5, t3
628 ADDV t1, acc3, acc3
629 SGTU t1, acc3, t4
630 ADDV t3, acc3, acc3
631 SGTU t3, acc3, t5
632 OR t4, t5, t3
633 ADDV t3, acc0, acc0
634
635 // Second reduction step
636 SLLV $32, acc1, t2
637 SRLV $32, acc1, t0
638 MOVV p256const1<>(SB), t6
639 MULV acc1, t6, t1
640 MULHVU acc1, t6, acc1
641 ADDV t2, acc2, acc2
642 SGTU t2, acc2, t3
643 ADDV t0, acc3, acc3
644 SGTU t0, acc3, t4
645 ADDV t3, acc3, acc3
646 SGTU t3, acc3, t5
647 OR t4, t5, t3
648 ADDV t1, acc0, acc0
649 SGTU t1, acc0, t4
650 ADDV t3, acc0, acc0
651 SGTU t3, acc0, t5
652 OR t4, t5, t3
653 ADDV t3, acc1, acc1
654
655 // Third reduction step
656 SLLV $32, acc2, t2
657 SRLV $32, acc2, t0
658 MOVV p256const1<>(SB), t6
659 MULV acc2, t6, t1
660 MULHVU acc2, t6, acc2
661 ADDV t2, acc3, acc3
662 SGTU t2, acc3, t3
663 ADDV t0, acc0, acc0
664 SGTU t0, acc0, t4
665 ADDV t3, acc0, acc0
666 SGTU t3, acc0, t5
667 OR t4, t5, t3
668 ADDV t1, acc1, acc1
669 SGTU t1, acc1, t4
670 ADDV t3, acc1, acc1
671 SGTU t3, acc1, t5
672 OR t4, t5, t3
673 ADDV t3, acc2, acc2
674
675 // Last reduction step
676 SLLV $32, acc3, t2
677 SRLV $32, acc3, t0
678 MOVV p256const1<>(SB), t6
679 MULV acc3, t6, t1
680 MULHVU acc3, t6, acc3
681 ADDV t2, acc0, acc0
682 SGTU t2, acc0, t3
683 ADDV t0, acc1, acc1
684 SGTU t0, acc1, t4
685 ADDV t3, acc1, acc1
686 SGTU t3, acc1, t5
687 OR t4, t5, t3
688 ADDV t1, acc2, acc2
689 SGTU t1, acc2, t4
690 ADDV t3, acc2, acc2
691 SGTU t3, acc2, t5
692 OR t4, t5, t3
693 ADDV t3, acc3, acc3
694
695 // Add bits [511:256] of the square result (acc4-acc7)
696 ADDV acc4, acc0, acc0
697 SGTU acc4, acc0, t0
698 ADDV acc5, acc1, acc1
699 SGTU acc5, acc1, t1
700 ADDV t0, acc1, acc1
701 SGTU t0, acc1, t2
702 OR t1, t2, t0
703 ADDV acc6, acc2, acc2
704 SGTU acc6, acc2, t1
705 ADDV t0, acc2, acc2
706 SGTU t0, acc2, t2
707 OR t1, t2, t0
708 ADDV acc7, acc3, acc3
709 SGTU acc7, acc3, t1
710 ADDV t0, acc3, acc3
711 SGTU t0, acc3, t2
712 OR t1, t2, acc4 // final overflow bit (0 or 1)
713
714 // Final conditional subtraction of P
715 SUBV $-1, acc0, t0
716 SGTU t0, acc0, t3
717 MOVV p256const0<>(SB), t5
718 SUBV t5, acc1, t1
719 SGTU t5, acc1, t4
720 SGTU t3, t1, t5
721 SUBV t3, t1, t1
722 OR t4, t5, t3
723 SUBV $0, acc2, t2
724 SGTU t3, t2, t4
725 SUBV t3, t2, t2
726 MOVV t4, t3
727 MOVV p256const1<>(SB), t6
728 SUBV t6, acc3, y0
729 SGTU y0, acc3, t4
730 SGTU t3, y0, t5
731 SUBV t3, y0, y0
732 OR t4, t5, t3
733 SUBV t3, acc4, acc4
734
735 // acc4 is -1 on borrow (result < P) and 0 otherwise; SGTU against
736 // the always-zero R0 turns it into a 0/1 flag, then into a mask.
737 SGTU acc4, R0, t3
738 SUBV t3, R0, t3 // t3 = borrow mask (0 or all-1s)
739 MOVV $-1, t4
740 XOR t3, t4, t4
741
742 AND t4, t0, t5
743 AND t3, acc0, t6
744 OR t5, t6, acc0
745
746 AND t4, t1, t5
747 AND t3, acc1, t6
748 OR t5, t6, acc1
749
750 AND t4, t2, t5
751 AND t3, acc2, t6
752 OR t5, t6, acc2
753
754 AND t4, y0, t5
755 AND t3, acc3, t6
756 OR t5, t6, acc3
757
758 MOVV acc0, y0
759 MOVV acc1, y1
760 MOVV acc2, y2
761 MOVV acc3, y3
762
763 RET
764
765 // Non-leaf: saves R1 and keeps n on the stack, because
766 // p256SqrInternal clobbers both R1 (CALL) and b_ptr (hlp0).
767 TEXT ·p256Sqr(SB),NOSPLIT,$16-24
768 MOVV R1, 8(R3) // save return address
769 MOVV res+0(FP), res_ptr
770 MOVV in+8(FP), a_ptr
771 MOVV n+16(FP), t0
772 MOVV t0, 16(R3) // spill n (b_ptr is clobbered by callee)
773
774 MOVV 0*8(a_ptr), x0
775 MOVV 1*8(a_ptr), x1
776 MOVV 2*8(a_ptr), x2
777 MOVV 3*8(a_ptr), x3
778
779 sqrLoop:
780 MOVV 16(R3), t0
781 SUBV $1, t0, t0
782 MOVV t0, 16(R3)
783 CALL p256SqrInternal<>(SB)
784 MOVV y0, x0
785 MOVV y1, x1
786 MOVV y2, x2
787 MOVV y3, x3
788 MOVV 16(R3), t0
789 BNE t0, R0, sqrLoop
790
791 MOVV y0, 0*8(res_ptr)
792 MOVV y1, 1*8(res_ptr)
793 MOVV y2, 2*8(res_ptr)
794 MOVV y3, 3*8(res_ptr)
795 MOVV 8(R3), R1 // restore return address
796 RET
797
798 TEXT p256MulInternal<>(SB),NOSPLIT,$0
799 // Initialize the upper half of the 512-bit accumulator.
800 // The first y0*x row only produces limbs acc0..acc4;
801 // acc5..acc7 must not contain stale register values.
802 MOVV $0, acc4
803 MOVV $0, acc5
804 MOVV $0, acc6
805 MOVV $0, acc7
806
807 // y0 * x
808 MULV y0, x0, acc0
809 MULHVU y0, x0, acc1
810
811 MULV y0, x1, t0
812 ADDV t0, acc1, acc1
813 SGTU t0, acc1, t1
814 MULHVU y0, x1, acc2
815 ADDV t1, acc2, acc2
816
817 MULV y0, x2, t0
818 ADDV t0, acc2, acc2
819 SGTU t0, acc2, t1
820 MULHVU y0, x2, acc3
821 ADDV t1, acc3, acc3
822
823 MULV y0, x3, t0
824 ADDV t0, acc3, acc3
825 SGTU t0, acc3, t1
826 MULHVU y0, x3, acc4
827 ADDV t1, acc4, acc4
828
829 // first reduction step
830 SLLV $32, acc0, t2
831 SRLV $32, acc0, t0
832 MOVV p256const1<>(SB), t6
833 MULV acc0, t6, t1
834 MULHVU acc0, t6, acc0
835
836 ADDV t2, acc1, acc1
837 SGTU t2, acc1, t3
838 ADDV t0, acc2, acc2
839 SGTU t0, acc2, t4
840 ADDV t3, acc2, acc2
841 SGTU t3, acc2, t3
842 OR t4, t3, t3
843 ADDV t1, acc3, acc3
844 SGTU t1, acc3, t4
845 ADDV t3, acc3, acc3
846 SGTU t3, acc3, t3
847 OR t4, t3, t3
848 ADDV t3, acc0, acc0 // acc0 = hi + carry: new pos-4 window limb
849
850 // y1 * x
851 MULV y1, x0, t0
852 ADDV t0, acc1, acc1
853 SGTU t0, acc1, t1
854 MULHVU y1, x0, t2
855
856 MULV y1, x1, t0
857 ADDV t0, acc2, acc2
858 SGTU t0, acc2, t3
859 ADDV t1, acc2, acc2
860 SGTU t1, acc2, t1
861 OR t3, t1, t1
862 MULHVU y1, x1, t4
863
864 MULV y1, x2, t0
865 ADDV t0, acc3, acc3
866 SGTU t0, acc3, t5
867 ADDV t1, acc3, acc3
868 SGTU t1, acc3, t1
869 OR t5, t1, t1
870 MULHVU y1, x2, t6
871
872 MULV y1, x3, t0
873 ADDV t0, acc4, acc4
874 SGTU t0, acc4, t5
875 ADDV t1, acc4, acc4
876 SGTU t1, acc4, t1
877 OR t5, t1, t1
878
879 // acc5 starts with the carry from the low-half chain.
880 // The high-half chain has its own carry, initially zero.
881 MOVV t1, acc5
882 MULHVU y1, x3, t0
883
884 // acc2 += high(y1*x0)
885 ADDV t2, acc2, acc2
886 SGTU t2, acc2, t3
887
888 // acc3 += high(y1*x1) + high-half carry
889 ADDV t4, acc3, acc3
890 SGTU t4, acc3, t5
891 ADDV t3, acc3, acc3
892 SGTU t3, acc3, t1
893 OR t5, t1, t3
894
895 // acc4 += high(y1*x2) + high-half carry
896 ADDV t6, acc4, acc4
897 SGTU t6, acc4, t5
898 ADDV t3, acc4, acc4
899 SGTU t3, acc4, t1
900 OR t5, t1, t3
901
902 // acc5 = low-half carry + high(y1*x3) + high-half carry
903 ADDV t0, acc5, acc5
904 SGTU t0, acc5, t5
905 ADDV t3, acc5, acc5
906 SGTU t3, acc5, t1
907 OR t5, t1, t3
908
909 // second reduction step
910 SLLV $32, acc1, t2
911 SRLV $32, acc1, t0
912 MOVV p256const1<>(SB), t6
913 MULV acc1, t6, t1
914 MULHVU acc1, t6, acc1
915
916 ADDV t2, acc2, acc2
917 SGTU t2, acc2, t3
918 ADDV t0, acc3, acc3
919 SGTU t0, acc3, t4
920 ADDV t3, acc3, acc3
921 SGTU t3, acc3, t3
922 OR t4, t3, t3
923 ADDV t1, acc0, acc0
924 SGTU t1, acc0, t4
925 ADDV t3, acc0, acc0
926 SGTU t3, acc0, t3
927 OR t4, t3, t3
928 ADDV t3, acc1, acc1
929
930 // y2 * x
931 MULV y2, x0, t0
932 ADDV t0, acc2, acc2
933 SGTU t0, acc2, t1
934 MULHVU y2, x0, t2
935
936 MULV y2, x1, t0
937 ADDV t0, acc3, acc3
938 SGTU t0, acc3, t3
939 MULHVU y2, x1, t4
940
941 ADDV t1, acc3, acc3
942 SGTU t1, acc3, t1
943 OR t3, t1, t1
944
945 MULV y2, x2, t0
946 ADDV t0, acc4, acc4
947 SGTU t0, acc4, t5
948 ADDV t1, acc4, acc4
949 SGTU t1, acc4, t1
950 OR t5, t1, t1
951 MULHVU y2, x2, t6
952
953 MULV y2, x3, t0
954 ADDV t0, acc5, acc5
955 SGTU t0, acc5, t5
956
957 ADDV t1, acc5, acc5
958 SGTU t1, acc5, t1
959 OR t5, t1, t1
960
961 // acc6 starts with the carry from the low-half chain.
962 // The high-half chain has its own carry, initially zero.
963 MOVV t1, acc6
964 MULHVU y2, x3, t0
965
966 // acc3 += high(y2*x0)
967 ADDV t2, acc3, acc3
968 SGTU t2, acc3, t3
969
970 // acc4 += high(y2*x1) + high-half carry
971 ADDV t4, acc4, acc4
972 SGTU t4, acc4, t5
973 ADDV t3, acc4, acc4
974 SGTU t3, acc4, t1
975 OR t5, t1, t3
976
977 // acc5 += high(y2*x2) + high-half carry
978 ADDV t6, acc5, acc5
979 SGTU t6, acc5, t5
980 ADDV t3, acc5, acc5
981 SGTU t3, acc5, t1
982 OR t5, t1, t3
983
984 // acc6 = low-half carry + high(y2*x3) + high-half carry
985 ADDV t0, acc6, acc6
986 SGTU t0, acc6, t5
987 ADDV t3, acc6, acc6
988 SGTU t3, acc6, t1
989 OR t5, t1, t3
990
991 // third reduction step
992 SLLV $32, acc2, t2
993 SRLV $32, acc2, t0
994 MOVV p256const1<>(SB), t6
995 MULV acc2, t6, t1
996 MULHVU acc2, t6, acc2
997
998 ADDV t2, acc3, acc3
999 SGTU t2, acc3, t3
1000 ADDV t0, acc0, acc0
1001 SGTU t0, acc0, t4
1002 ADDV t3, acc0, acc0
1003 SGTU t3, acc0, t3
1004 OR t4, t3, t3
1005 ADDV t1, acc1, acc1
1006 SGTU t1, acc1, t4
1007 ADDV t3, acc1, acc1
1008 SGTU t3, acc1, t3
1009 OR t4, t3, t3
1010 ADDV t3, acc2, acc2
1011
1012 // y3 * x
1013 MULV y3, x0, t0
1014 ADDV t0, acc3, acc3
1015 SGTU t0, acc3, t1
1016 MULHVU y3, x0, t2
1017
1018 MULV y3, x1, t0
1019 ADDV t0, acc4, acc4
1020 SGTU t0, acc4, t3
1021 MULHVU y3, x1, t4
1022
1023 ADDV t1, acc4, acc4
1024 SGTU t1, acc4, t1
1025 OR t3, t1, t1
1026
1027 MULV y3, x2, t0
1028 ADDV t0, acc5, acc5
1029 SGTU t0, acc5, t5
1030 ADDV t1, acc5, acc5
1031 SGTU t1, acc5, t1
1032 OR t5, t1, t1
1033 MULHVU y3, x2, t6
1034
1035 MULV y3, x3, t0
1036 ADDV t0, acc6, acc6
1037 SGTU t0, acc6, t5
1038
1039 ADDV t1, acc6, acc6
1040 SGTU t1, acc6, t1
1041 OR t5, t1, t1
1042
1043 // acc7 starts with the carry from the low-half chain.
1044 // The high-half chain has its own carry, initially zero.
1045 MOVV t1, acc7
1046 MULHVU y3, x3, t0
1047
1048 // acc4 += high(y3*x0)
1049 ADDV t2, acc4, acc4
1050 SGTU t2, acc4, t3
1051
1052 // acc5 += high(y3*x1) + high-half carry
1053 ADDV t4, acc5, acc5
1054 SGTU t4, acc5, t5
1055 ADDV t3, acc5, acc5
1056 SGTU t3, acc5, t1
1057 OR t5, t1, t3
1058
1059 // acc6 += high(y3*x2) + high-half carry
1060 ADDV t6, acc6, acc6
1061 SGTU t6, acc6, t5
1062 ADDV t3, acc6, acc6
1063 SGTU t3, acc6, t1
1064 OR t5, t1, t3
1065
1066 // acc7 = low-half carry + high(y3*x3) + high-half carry
1067 ADDV t0, acc7, acc7
1068 SGTU t0, acc7, t5
1069 ADDV t3, acc7, acc7
1070 SGTU t3, acc7, t1
1071 OR t5, t1, t3
1072 // carry beyond acc7 is outside the 512-bit product and is discarded.
1073
1074 // fourth reduction step
1075 SLLV $32, acc3, t2
1076 SRLV $32, acc3, t0
1077 MOVV p256const1<>(SB), t6
1078 MULV acc3, t6, t1
1079 MULHVU acc3, t6, acc3
1080
1081 ADDV t2, acc0, acc0
1082 SGTU t2, acc0, t3
1083 ADDV t0, acc1, acc1
1084 SGTU t0, acc1, t4
1085 ADDV t3, acc1, acc1
1086 SGTU t3, acc1, t3
1087 OR t4, t3, t3
1088 ADDV t1, acc2, acc2
1089 SGTU t1, acc2, t4
1090 ADDV t3, acc2, acc2
1091 SGTU t3, acc2, t3
1092 OR t4, t3, t3
1093 ADDV t3, acc3, acc3
1094
1095 // fold high half [511:256]
1096 ADDV acc4, acc0, acc0
1097 SGTU acc4, acc0, t0
1098 ADDV acc5, acc1, acc1
1099 SGTU acc5, acc1, t1
1100 ADDV t0, acc1, acc1
1101 SGTU t0, acc1, t0
1102 OR t1, t0, t0
1103
1104 ADDV acc6, acc2, acc2
1105 SGTU acc6, acc2, t1
1106 ADDV t0, acc2, acc2
1107 SGTU t0, acc2, t0
1108 OR t1, t0, t0
1109
1110 ADDV acc7, acc3, acc3
1111 SGTU acc7, acc3, t1
1112 ADDV t0, acc3, acc3
1113 SGTU t0, acc3, t0
1114 OR t1, t0, t0
1115
1116 MOVV t0, acc4 // final overflow flag (0 or 1)
1117
1118 // conditional subtraction of P (mask select)
1119 // compute borrows
1120 MOVV $-1, t6
1121 SGTU t6, acc0, t5 // borrow0 = 1 if acc0 < 0xFFFFFFFFFFFFFFFF
1122 ADDV $1, acc0, t0 // d0 = acc0 + 1
1123
1124 MOVV p256const0<>(SB), t6
1125 SUBV t6, acc1, t1
1126 SGTU t6, acc1, t4
1127 MOVV t1, t6
1128 SUBV t5, t1, t1
1129 SGTU t1, t6, t6
1130 OR t4, t6, t5 // borrow1
1131
1132 SUBV $0, acc2, t2
1133 SGTU $0, acc2, t4 // always 0
1134 MOVV t2, t6
1135 SUBV t5, t2, t2
1136 SGTU t2, t6, t6
1137 OR t4, t6, t5
1138
1139 MOVV p256const1<>(SB), t6
1140 SUBV t6, acc3, t3
1141 SGTU t6, acc3, t4
1142 MOVV t3, t6
1143 SUBV t5, t3, t3
1144 SGTU t3, t6, t6
1145 OR t4, t6, t5 // final borrow in t5 (0 or 1)
1146
1147 // Fold the extra overflow limb into the borrow chain:
1148 // acc4 -= borrow(acc0..acc3).
1149 // acc4=1, no borrow -> acc4 stays 1, still >= P, subtract P
1150 // acc4=0, borrow -> acc4 becomes -1, value < P, keep original
1151 // acc4=1, borrow -> acc4 becomes 0, exactly cancels, subtract P
1152 // acc4=0, no borrow -> acc4 stays 0, subtract P
1153 SUBV t5, acc4, acc4
1154
1155 // acc4 != 0 means borrow occurred (value < P): keep acc0..acc3.
1156 // acc4 == 0 means no borrow (value >= P): select the subtraction t0..t3.
1157 SGTU acc4, R0, t5
1158 SUBV t5, R0, t5 // t5 = borrow mask: all-1s keeps original
1159 MOVV $-1, t6
1160 XOR t5, t6, t6 // t6 = no-borrow mask: all-1s selects diff
1161
1162 // borrow==0 (acc >= P) -> take t0..t3; borrow==1 (acc < P) -> keep acc0..acc3
1163 AND t6, t0, t4
1164 AND t5, acc0, acc4
1165 OR t4, acc4, y0
1166
1167 AND t6, t1, t4
1168 AND t5, acc1, acc4
1169 OR t4, acc4, y1
1170
1171 AND t6, t2, t4
1172 AND t5, acc2, acc4
1173 OR t4, acc4, y2
1174
1175 AND t6, t3, t4
1176 AND t5, acc3, acc4
1177 OR t4, acc4, y3
1178
1179 RET
1180
1181 // func p256Mul(res, in1, in2 *p256Element)
1182 TEXT ·p256Mul(SB),NOSPLIT,$16-24
1183 MOVV R1, 8(R3) // save return address
1184 MOVV res+0(FP), res_ptr
1185 MOVV in1+8(FP), a_ptr
1186 MOVV in2+16(FP), b_ptr
1187
1188 MOVV 0*8(a_ptr), x0
1189 MOVV 1*8(a_ptr), x1
1190 MOVV 2*8(a_ptr), x2
1191 MOVV 3*8(a_ptr), x3
1192
1193 MOVV 0*8(b_ptr), y0
1194 MOVV 1*8(b_ptr), y1
1195 MOVV 2*8(b_ptr), y2
1196 MOVV 3*8(b_ptr), y3
1197
1198 CALL p256MulInternal<>(SB)
1199
1200 MOVV y0, 0*8(res_ptr)
1201 MOVV y1, 1*8(res_ptr)
1202 MOVV y2, 2*8(res_ptr)
1203 MOVV y3, 3*8(res_ptr)
1204 MOVV 8(R3), R1 // restore return address
1205 RET
1206
1207 #define p256MulBy2Inline \
1208 /* x0..x3 = y0..y3 + y0..y3 (double), carry out -> hlp0 */ \
1209 ADDV y0, y0, x0 ;\
1210 SGTU y0, x0, t0 ;\
1211 \
1212 ADDV y1, y1, t1 ;\
1213 SGTU y1, t1, t2 ;\
1214 ADDV t0, t1, x1 ;\
1215 SGTU t0, x1, t3 ;\
1216 OR t2, t3, t0 ;\
1217 \
1218 ADDV y2, y2, t1 ;\
1219 SGTU y2, t1, t2 ;\
1220 ADDV t0, t1, x2 ;\
1221 SGTU t0, x2, t3 ;\
1222 OR t2, t3, t0 ;\
1223 \
1224 ADDV y3, y3, t1 ;\
1225 SGTU y3, t1, t2 ;\
1226 ADDV t0, t1, x3 ;\
1227 SGTU t0, x3, t3 ;\
1228 OR t2, t3, hlp0 ;\
1229 \
1230 /* t0..t3 = x0..x3 - P (4-limb borrow chain, same pattern as in p256MulInternal) */ \
1231 MOVV $-1, t6 ;\
1232 SGTU t6, x0, t5 ;\
1233 ADDV $1, x0, t0 ;\
1234 \
1235 MOVV p256const0<>(SB), t6 ;\
1236 SUBV t6, x1, t1 ;\
1237 SGTU t6, x1, t4 ;\
1238 MOVV t1, t6 ;\
1239 SUBV t5, t1, t1 ;\
1240 SGTU t5, t6, t6 ;\ /* borrow from subtracting borrow_in */
1241 OR t4, t6, t5 ;\
1242 \
1243 SUBV $0, x2, t2 ;\
1244 SGTU $0, x2, t4 ;\
1245 MOVV t2, t6 ;\
1246 SUBV t5, t2, t2 ;\
1247 SGTU t5, t6, t6 ;\
1248 OR t4, t6, t5 ;\
1249 \
1250 MOVV p256const1<>(SB), t6 ;\
1251 SUBV t6, x3, t3 ;\
1252 SGTU t6, x3, t4 ;\
1253 MOVV t3, t6 ;\
1254 SUBV t5, t3, t3 ;\
1255 SGTU t5, t6, t6 ;\
1256 OR t4, t6, t5 ;\
1257 \
1258 /* fold the doubling carry bit into the borrow chain */ \
1259 SUBV t5, hlp0, hlp0 ;\ /* hlp0 = ovf - borrow */
1260 SRAV $63, hlp0, t5 ;\
1261 MOVV $-1, t6 ;\
1262 XOR t5, t6, t6 ;\
1263 \
1264 /* select: no borrow (>= P) -> t0..t3; borrow (< P) -> original x0..x3 */ \
1265 AND t6, t0, t4 ;\
1266 AND t5, x0, hlp0 ;\
1267 OR t4, hlp0, x0 ;\
1268 \
1269 AND t6, t1, t4 ;\
1270 AND t5, x1, hlp0 ;\
1271 OR t4, hlp0, x1 ;\
1272 \
1273 AND t6, t2, t4 ;\
1274 AND t5, x2, hlp0 ;\
1275 OR t4, hlp0, x2 ;\
1276 \
1277 AND t6, t3, t4 ;\
1278 AND t5, x3, hlp0 ;\
1279 OR t4, hlp0, x3 ;
1280
1281 #define y2in(off) (8 + 32*0 + off)(R3)
1282 #define s2v(off) (8 + 32*1 + off)(R3)
1283 #define z1sqr(off) (8 + 32*2 + off)(R3)
1284 #define hv(off) (8 + 32*3 + off)(R3)
1285 #define rv(off) (8 + 32*4 + off)(R3)
1286 #define hsqr(off) (8 + 32*5 + off)(R3)
1287 #define rsqr(off) (8 + 32*6 + off)(R3)
1288 #define hcub(off) (8 + 32*7 + off)(R3)
1289
1290 /*
1291 * slot 8:
1292 * +0 : sel | (zero << 1)
1293 * +8 : normalized sign
1294 *
1295 * Both values need to survive CALLs, so keep them on the stack.
1296 */
1297 #define flagbase(off) (8 + 32*8 + off)(R3)
1298 #define selflag(off) flagbase(off)
1299 #define signflag(off) flagbase(8 + off)
1300
1301 // z3 staging slot: deferred so that res==in1 aliasing cannot clobber z1in
1302 // before its last read.
1303 #define z3save(off) (8 + 32*12 + off)(R3)
1304
1305 #define x1in(off) (off)(a_ptr)
1306 #define y1in(off) (off+32)(a_ptr)
1307 #define z1in(off) (off+64)(a_ptr)
1308 #define x2in(off) (off)(b_ptr)
1309 #define z2in(off) (off + 64)(b_ptr)
1310 #define x3out(off) (off)(res_ptr)
1311 #define y3out(off) (off + 32)(res_ptr)
1312 #define z3out(off) (off + 64)(res_ptr)
1313 #define y2inptr(off) (off+32)(b_ptr)
1314
1315 #define LDx(src) \
1316 MOVV src(0*8),x0; \
1317 MOVV src(1*8),x1; \
1318 MOVV src(2*8),x2; \
1319 MOVV src(3*8),x3
1320
1321 // LDy deliberately loads limb 1 into t4 first because y1 must not alias
1322 // either a_ptr or b_ptr. All source limbs are loaded before y1 is written.
1323 #define LDy(src) \
1324 MOVV src(0*8),y0; \
1325 MOVV src(1*8),t4; \
1326 MOVV src(2*8),y2; \
1327 MOVV src(3*8),y3; \
1328 MOVV t4, y1
1329
1330 #define STx(dst) MOVV x0,dst(0*8); MOVV x1,dst(1*8); MOVV x2,dst(2*8); MOVV x3,dst(3*8)
1331 #define STy(dst) MOVV y0,dst(0*8); MOVV y1,dst(1*8); MOVV y2,dst(2*8); MOVV y3,dst(3*8)
1332
1333 #define RELOAD_PTRS \
1334 MOVV in1+8(FP), a_ptr ;\
1335 MOVV in2+16(FP), b_ptr
1336
1337 // func p256PointAddAffineAsm(res, in1 *P256Point, in2 *p256AffinePoint, sign, sel, zero int)
1338 TEXT ·p256PointAddAffineAsm(SB),NOSPLIT,$432-48
1339 MOVV R1, (8+32*9)(R3) // save return address across CALLs
1340 // sign/sel/zero cannot live in hlp0(=b_ptr)/y1(=a_ptr); use t5/t6 first.
1341 MOVV sign+24(FP), t5 // t5 = sign
1342 MOVV sel+32(FP), t6 // t6 = sel
1343 MOVV zero+40(FP), t0 // t0 = zero
1344
1345 // Normalize sel/zero to 0/1 and combine them into a bitmask saved in t1.
1346 SGTU t6, R0, t2 // t2 = (sel != 0) ? 1 : 0
1347 SGTU t0, R0, t3 // t3 = (zero != 0) ? 1 : 0
1348 SLLV $1, t3, t3
1349 OR t2, t3, t1 // t1 = sel_norm | (zero_norm << 1)
1350 MOVV t1, selflag(0) // spill to stack; must survive CALLs below
1351
1352 // Normalize sign to 0/1 and spill it for the conditional select below.
1353 SGTU t5, R0, t4
1354 MOVV t4, signflag(0) // sign must survive CALLs; store on stack
1355
1356 RELOAD_PTRS // ensure a_ptr/b_ptr are clean
1357
1358 // Negate y2in based on sign: compute P - y2in via p256SubInternal
1359 // p256SubInternal computes diff = y - x (result in x0..x3), so:
1360 // x0..x3 = y2in (value to subtract)
1361 // y0..y3 = P
1362 MOVV y2inptr(0*8), x0
1363 MOVV y2inptr(1*8), x1
1364 MOVV y2inptr(2*8), x2
1365 MOVV y2inptr(3*8), x3
1366
1367 MOVV $-1, y0
1368 MOVV p256const0<>(SB), y1
1369 MOVV $0, y2
1370 MOVV p256const1<>(SB), y3
1371
1372 CALL p256SubInternal<>(SB) // x0..x3 = P - y2in
1373 RELOAD_PTRS // a_ptr/b_ptr(=hlp0)/y1 clobbered by the call
1374
1375 // reload original y2in (y0..y3 were consumed as the P operand above)
1376 MOVV y2inptr(0*8), y0
1377 MOVV y2inptr(1*8), y1
1378 MOVV y2inptr(2*8), y2
1379 MOVV y2inptr(3*8), y3
1380
1381 // Restore normalized sign from the stack.
1382 // mask = 0 (sign==0, keep original) or all-ones (sign!=0, use negated)
1383 MOVV signflag(0), t4 // restore normalized sign
1384 SUBV t4, R0, t2 // t2 = mask
1385 MOVV $-1, t3
1386 XOR t2, t3, t3 // t3 = notmask
1387
1388 AND t3, y0, acc0
1389 AND t2, x0, acc4
1390 OR acc0, acc4, y0
1391
1392 AND t3, y1, acc0
1393 AND t2, x1, acc4
1394 OR acc0, acc4, y1
1395
1396 AND t3, y2, acc0
1397 AND t2, x2, acc4
1398 OR acc0, acc4, y2
1399
1400 AND t3, y3, acc0
1401 AND t2, x3, acc4
1402 OR acc0, acc4, y3
1403
1404 STy(y2in)
1405
1406 // Begin point add
1407 RELOAD_PTRS
1408
1409 LDx(z1in)
1410 CALL p256SqrInternal<>(SB) // z1^2 -- a_ptr/b_ptr(=hlp0)/y1 clobbered
1411 STy(z1sqr)
1412 RELOAD_PTRS
1413
1414 LDx(x2in)
1415 LDy(z1sqr) // y = z1^2 (RELOAD_PTRS clobbered y1)
1416 CALL p256MulInternal<>(SB) // u2 = x2 * z1^2
1417 MOVV y0, rsqr(0*8)
1418 MOVV y1, rsqr(1*8)
1419 MOVV y2, rsqr(2*8)
1420 MOVV y3, rsqr(3*8)
1421 RELOAD_PTRS
1422
1423 LDx(x1in)
1424 LDy(rsqr)
1425 CALL p256SubInternal<>(SB) // h = u2 - x1
1426 STx(hv)
1427 RELOAD_PTRS
1428
1429 LDy(z1in)
1430 CALL p256MulInternal<>(SB) // z3 = h * z1
1431 STy(s2v)
1432 RELOAD_PTRS
1433
1434 // Conditional z3 select: bit0 clear -> z1; bit1 clear -> 1.
1435 // Read z1in first (needs a_ptr; restored by RELOAD_PTRS above).
1436 MOVV z1in(0*8), acc0
1437 MOVV z1in(1*8), acc1
1438 MOVV z1in(2*8), acc2
1439 MOVV z1in(3*8), acc3
1440
1441 // Reload y = z3 (s2v is an R3-relative slot, independent of a_ptr;
1442 // y1 is written last, and no a_ptr dereference follows).
1443 MOVV s2v(0*8), y0
1444 MOVV s2v(1*8), t4
1445 MOVV s2v(2*8), y2
1446 MOVV s2v(3*8), y3
1447 MOVV t4, y1
1448
1449 MOVV selflag(0), t1 // reload bitmask; t1 may have been clobbered
1450 AND $1, t1, t2
1451 NEGV t2, t2 // 0/1 -> 0/all-ones mask
1452 MOVV $-1, t3
1453 XOR t2, t3, t3 // t3 = ~mask
1454 AND t2, y0, y0; AND t3, acc0, acc0; OR y0, acc0, y0
1455 AND t2, y1, y1; AND t3, acc1, acc1; OR y1, acc1, y1
1456 AND t2, y2, y2; AND t3, acc2, acc2; OR y2, acc2, y2
1457 AND t2, y3, y3; AND t3, acc3, acc3; OR y3, acc3, y3
1458
1459 MOVV p256one<>+0x00(SB), acc0
1460 MOVV p256one<>+0x08(SB), acc1
1461 MOVV p256one<>+0x10(SB), acc2
1462 MOVV p256one<>+0x18(SB), acc3
1463 MOVV selflag(0), t1
1464 AND $2, t1, t2
1465 SRLV $1, t2, t2
1466 NEGV t2, t2
1467 MOVV $-1, t3
1468 XOR t2, t3, t3
1469 AND t2, y0, y0; AND t3, acc0, acc0; OR y0, acc0, y0
1470 AND t2, y1, y1; AND t3, acc1, acc1; OR y1, acc1, y1
1471 AND t2, y2, y2; AND t3, acc2, acc2; OR y2, acc2, y2
1472 AND t2, y3, y3; AND t3, acc3, acc3; OR y3, acc3, y3
1473
1474 // Save the selected z3 before loading the next temporary.
1475 // It must not be written to res.z yet: when res==in1, the later
1476 // LDx(z1in) below still needs the old z1 value.
1477 STy(z3save) // y0..y3 -> z3save
1478
1479 MOVV in1+8(FP), a_ptr // y is spilled; restore a_ptr to read in1
1480 LDx(z1in) // x = z1 (x previously held h)
1481 LDy(z1sqr)
1482 CALL p256MulInternal<>(SB) // z1^3
1483 MOVV y0, rsqr(0*8)
1484 MOVV y1, rsqr(1*8)
1485 MOVV y2, rsqr(2*8)
1486 MOVV y3, rsqr(3*8)
1487 RELOAD_PTRS
1488
1489 LDx(y2in)
1490 LDy(rsqr)
1491 CALL p256MulInternal<>(SB) // s2 = y2in * z1^3
1492 STy(s2v)
1493 RELOAD_PTRS
1494
1495 LDx(y1in)
1496 LDy(s2v) // y = s2
1497 CALL p256SubInternal<>(SB) // r = s2 - y1
1498 STx(rv)
1499 RELOAD_PTRS
1500
1501 CALL p256SqrInternal<>(SB) // rsqr = r^2
1502 STy(rsqr)
1503 RELOAD_PTRS
1504
1505 LDx(hv)
1506 CALL p256SqrInternal<>(SB) // hsqr = h^2
1507 STy(hsqr)
1508 RELOAD_PTRS
1509
1510 LDy(hsqr) // y = h^2
1511 CALL p256MulInternal<>(SB) // hcub = h^3
1512 STy(hcub)
1513 RELOAD_PTRS
1514
1515 LDx(y1in)
1516 LDy(hcub) // y = hcub
1517 CALL p256MulInternal<>(SB) // s2' = y1 * hcub
1518 STy(s2v)
1519 RELOAD_PTRS
1520
1521 MOVV hsqr(0*8), x0
1522 MOVV hsqr(1*8), x1
1523 MOVV hsqr(2*8), x2
1524 MOVV hsqr(3*8), x3
1525 MOVV x1in(0*8), y0
1526 MOVV x1in(1*8), t4
1527 MOVV x1in(2*8), y2
1528 MOVV x1in(3*8), y3
1529 MOVV t4, y1
1530 CALL p256MulInternal<>(SB) // u1' = x1 * hsqr
1531 STy(hv) // overwrite spilled h
1532 // NOTE: MulBy2Inline reads y1 which aliases a_ptr; it must run
1533 // BEFORE RELOAD_PTRS. LDy(rsqr) only needs the stack pointer.
1534 p256MulBy2Inline // u1'2 = 2*u1' (y regs still hold u1')
1535
1536 LDy(rsqr)
1537 CALL p256SubInternal<>(SB) // rsqr - u1'2
1538 RELOAD_PTRS
1539
1540 MOVV x0, y0
1541 MOVV x1, y1
1542 MOVV x2, y2
1543 MOVV x3, y3
1544
1545 LDx(hcub)
1546 CALL p256SubInternal<>(SB) // x3 = previous result - hcub
1547 RELOAD_PTRS
1548
1549 // Conditional x3 select.
1550 MOVV x1in(0*8), acc0
1551 MOVV x1in(1*8), acc1
1552 MOVV x1in(2*8), acc2
1553 MOVV x1in(3*8), acc3
1554 MOVV selflag(0), t1
1555 AND $1, t1, t2
1556 NEGV t2, t2
1557 MOVV $-1, t3
1558 XOR t2, t3, t3
1559 AND t2, x0, x0; AND t3, acc0, acc0; OR x0, acc0, x0
1560 AND t2, x1, x1; AND t3, acc1, acc1; OR x1, acc1, x1
1561 AND t2, x2, x2; AND t3, acc2, acc2; OR x2, acc2, x2
1562 AND t2, x3, x3; AND t3, acc3, acc3; OR x3, acc3, x3
1563
1564 RELOAD_PTRS
1565 MOVV x2in(0*8), acc0
1566 MOVV x2in(1*8), acc1
1567 MOVV x2in(2*8), acc2
1568 MOVV x2in(3*8), acc3
1569 MOVV selflag(0), t1
1570 AND $2, t1, t2
1571 SRLV $1, t2, t2
1572 NEGV t2, t2
1573 MOVV $-1, t3
1574 XOR t2, t3, t3
1575 AND t2, x0, x0; AND t3, acc0, acc0; OR x0, acc0, x0
1576 AND t2, x1, x1; AND t3, acc1, acc1; OR x1, acc1, x1
1577 AND t2, x2, x2; AND t3, acc2, acc2; OR x2, acc2, x2
1578 AND t2, x3, x3; AND t3, acc3, acc3; OR x3, acc3, x3
1579
1580 RELOAD_PTRS
1581 MOVV res+0(FP), t0
1582 MOVV x0, 0*8(t0)
1583 MOVV x1, 1*8(t0)
1584 MOVV x2, 2*8(t0)
1585 MOVV x3, 3*8(t0)
1586
1587 MOVV 0*8(t0), x0
1588 MOVV 1*8(t0), x1
1589 MOVV 2*8(t0), x2
1590 MOVV 3*8(t0), x3
1591 LDy(hv) // u1'
1592 CALL p256SubInternal<>(SB) // tmp = u1' - x3
1593 RELOAD_PTRS
1594
1595 MOVV rv(0*8), y0
1596 MOVV rv(1*8), y1
1597 MOVV rv(2*8), y2
1598 MOVV rv(3*8), y3
1599 CALL p256MulInternal<>(SB) // tmp2 = r * tmp
1600 MOVV y0, rv(0*8)
1601 MOVV y1, rv(1*8)
1602 MOVV y2, rv(2*8)
1603 MOVV y3, rv(3*8)
1604 RELOAD_PTRS
1605
1606 LDx(s2v)
1607 LDy(rv)
1608 CALL p256SubInternal<>(SB) // y3 = tmp2 - s2'
1609 RELOAD_PTRS
1610
1611 // Conditional y3 select.
1612 MOVV y1in(0*8), acc0
1613 MOVV y1in(1*8), acc1
1614 MOVV y1in(2*8), acc2
1615 MOVV y1in(3*8), acc3
1616 MOVV selflag(0), t1
1617 AND $1, t1, t2
1618 NEGV t2, t2
1619 MOVV $-1, t3
1620 XOR t2, t3, t3
1621 AND t2, x0, x0; AND t3, acc0, acc0; OR x0, acc0, x0
1622 AND t2, x1, x1; AND t3, acc1, acc1; OR x1, acc1, x1
1623 AND t2, x2, x2; AND t3, acc2, acc2; OR x2, acc2, x2
1624 AND t2, x3, x3; AND t3, acc3, acc3; OR x3, acc3, x3
1625
1626 MOVV y2in(0*8), acc0
1627 MOVV y2in(1*8), acc1
1628 MOVV y2in(2*8), acc2
1629 MOVV y2in(3*8), acc3
1630 MOVV selflag(0), t1
1631 AND $2, t1, t2
1632 SRLV $1, t2, t2
1633 NEGV t2, t2
1634 MOVV $-1, t3
1635 XOR t2, t3, t3
1636 AND t2, x0, x0; AND t3, acc0, acc0; OR x0, acc0, x0
1637 AND t2, x1, x1; AND t3, acc1, acc1; OR x1, acc1, x1
1638 AND t2, x2, x2; AND t3, acc2, acc2; OR x2, acc2, x2
1639 AND t2, x3, x3; AND t3, acc3, acc3; OR x3, acc3, x3
1640
1641 MOVV res+0(FP), t0
1642 MOVV x0, 4*8(t0)
1643 MOVV x1, 5*8(t0)
1644 MOVV x2, 6*8(t0)
1645 MOVV x3, 7*8(t0)
1646
1647 // Deferred res.z write: all z1in reads are complete at this point.
1648 LDx(z3save)
1649 MOVV x0, 8*8(t0)
1650 MOVV x1, 9*8(t0)
1651 MOVV x2, 10*8(t0)
1652 MOVV x3, 11*8(t0)
1653
1654 MOVV (8+32*9)(R3), R1 // restore return address
1655 RET
1656
1657 // Stack temporaries for PointDouble (R3-relative, avoids the saved LR).
1658 #define zsqrv(off) (8 + 32*0 + off)(R3)
1659 #define mv(off) (8 + 32*1 + off)(R3)
1660 #define h2v(off) (8 + 32*2 + off)(R3)
1661 #define sv(off) (8 + 32*3 + off)(R3)
1662 #define tmpv(off) (8 + 32*4 + off)(R3)
1663 #define y3tmp(off) (8 + 32*5 + off)(R3)
1664
1665 // mod-P halving: y0..y3 = (y0..y3) / 2 mod P
1666 // If even, shift right by 1; if odd, add P first, then shift right by 1.
1667 #define p256HalveInline \
1668 ADDV $-1, y0, t0 ;\
1669 SGTU $-1, t0, t4 ;\
1670 MOVV p256const0<>(SB), t5 ;\
1671 ADDV t5, y1, t1 ;\
1672 SGTU t5, t1, t5 ;\
1673 ADDV t4, t1, t1 ;\
1674 SGTU t4, t1, t6 ;\
1675 OR t5, t6, t4 ;\
1676 ADDV $0, y2, t2 ;\
1677 ADDV t4, t2, t2 ;\
1678 SGTU t4, t2, t4 ;\
1679 MOVV p256const1<>(SB), t5 ;\
1680 ADDV t5, y3, t3 ;\
1681 SGTU t5, t3, t5 ;\
1682 ADDV t4, t3, t3 ;\
1683 SGTU t4, t3, t6 ;\
1684 OR t5, t6, hlp0 ;\
1685 \
1686 AND $1, y0, t4 ;\
1687 NEGV t4, t4 ;\ /* t4 = 0 (even) / all-ones (odd) */
1688 MOVV $-1, t5 ;\
1689 XOR t4, t5, t5 ;\ /* t5 = notmask */
1690 AND t5, y0, acc0; AND t4, t0, acc4; OR acc0, acc4, t0 ;\
1691 AND t5, y1, acc0; AND t4, t1, acc4; OR acc0, acc4, t1 ;\
1692 AND t5, y2, acc0; AND t4, t2, acc4; OR acc0, acc4, t2 ;\
1693 AND t5, y3, acc0; AND t4, t3, acc4; OR acc0, acc4, t3 ;\
1694 AND t4, hlp0, hlp0 ;\
1695 \
1696 SRLV $1, t0, y0 ;\
1697 SLLV $63, t1, acc0 ;\
1698 OR acc0, y0, y0 ;\
1699 SRLV $1, t1, y1 ;\
1700 SLLV $63, t2, acc0 ;\
1701 OR acc0, y1, y1 ;\
1702 SRLV $1, t2, y2 ;\
1703 SLLV $63, t3, acc0 ;\
1704 OR acc0, y2, y2 ;\
1705 SRLV $1, t3, y3 ;\
1706 SLLV $63, hlp0, acc0 ;\
1707 OR acc0, y3, y3
1708
1709 // x = (x + y) mod P; result written back in place to x0..x3
1710 // y0..y3 are read but not written; their contents are preserved
1711 #define p256AddInline \
1712 ADDV y0, x0, x0 ;\
1713 SGTU y0, x0, hlp0 ;\ /* hlp0 = limb0 carry */
1714 ADDV y1, x1, x1 ;\
1715 SGTU y1, x1, t4 ;\
1716 ADDV hlp0, x1, x1 ;\
1717 SGTU hlp0, x1, t5 ;\
1718 OR t4, t5, hlp0 ;\
1719 ADDV y2, x2, x2 ;\
1720 SGTU y2, x2, t4 ;\
1721 ADDV hlp0, x2, x2 ;\
1722 SGTU hlp0, x2, t5 ;\
1723 OR t4, t5, hlp0 ;\
1724 ADDV y3, x3, x3 ;\
1725 SGTU y3, x3, t4 ;\
1726 ADDV hlp0, x3, x3 ;\
1727 SGTU hlp0, x3, t5 ;\
1728 OR t4, t5, hlp0 ;\ /* hlp0 = final overflow (0/1); 1 means x+y >= 2^256 */
1729 \
1730 SUBV $-1, x0, t0 ;\ /* t0 = x0 - P0 (P0 = -1, i.e. 0xFFFF...FFFF) */
1731 SGTU $-1, x0, t4 ;\ /* t4 = borrow0 = (x0 < P0) */
1732 MOVV p256const0<>(SB), t5 ;\
1733 SUBV t5, x1, t1 ;\
1734 SGTU t5, x1, t5 ;\
1735 SUBV t4, t1, t1 ;\
1736 SGTU t4, t1, t6 ;\
1737 OR t5, t6, t4 ;\
1738 SUBV $0, x2, t2 ;\
1739 SGTU $0, x2, t5 ;\
1740 SUBV t4, t2, t2 ;\
1741 SGTU t4, t2, t6 ;\
1742 OR t5, t6, t4 ;\
1743 MOVV p256const1<>(SB), t5 ;\
1744 SUBV t5, x3, t3 ;\
1745 SGTU t5, x3, t5 ;\
1746 SUBV t4, t3, t3 ;\
1747 SGTU t4, t3, t6 ;\
1748 OR t5, t6, t4 ;\ /* t4 = 4-limb borrow (1 means x < P, no subtraction needed) */
1749 \
1750 SUBV t4, hlp0, hlp0 ;\ /* hlp0 - t4: 1/0 (subtract P) or -1 (no subtract) */
1751 SRAV $63, hlp0, t5 ;\ /* t5 = all-1s (no subtract) or all-0s (subtract) */
1752 MOVV $-1, t6 ;\
1753 XOR t5, t6, t6 ;\ /* t6 = notmask: all-1s (subtract) or all-0s (no subtract) */
1754 \
1755 AND t6, t0, hlp0 ;\
1756 AND t5, x0, x0 ;\
1757 OR hlp0, x0, x0 ;\
1758 AND t6, t1, hlp0 ;\
1759 AND t5, x1, x1 ;\
1760 OR hlp0, x1, x1 ;\
1761 AND t6, t2, hlp0 ;\
1762 AND t5, x2, x2 ;\
1763 OR hlp0, x2, x2 ;\
1764 AND t6, t3, hlp0 ;\
1765 AND t5, x3, x3 ;\
1766 OR hlp0, x3, x3
1767
1768 // func p256PointDoubleAsm(res, in *P256Point)
1769 TEXT ·p256PointDoubleAsm(SB),NOSPLIT,$216-16
1770 MOVV R1, (8+32*6)(R3) // save return address; NOSPLIT => no auto-save
1771 MOVV res+0(FP), res_ptr
1772 MOVV in+8(FP), a_ptr
1773
1774 // zsqr = Z1^2
1775 LDx(z1in)
1776 CALL p256SqrInternal<>(SB)
1777 STy(zsqrv)
1778 MOVV in+8(FP), a_ptr
1779
1780 // m_pre = X1 + zsqr (x = X1, y = zsqr)
1781 LDx(x1in)
1782 LDy(zsqrv)
1783 p256AddInline // x0..x3 = x + y = X1 + zsqr
1784 STx(mv) // stash m_pre; x1in/zsqr are reused below
1785
1786 // a_ptr is caller-clobbered by previous CALLs.
1787 MOVV in+8(FP), a_ptr
1788
1789 // z3 = 2*(Y1*Z1) -> res.z
1790 LDx(z1in)
1791 LDy(y1in)
1792 CALL p256MulInternal<>(SB)
1793 p256MulBy2Inline
1794 STx(z3out)
1795 MOVV in+8(FP), a_ptr // reload AFTER y is consumed, BEFORE next in1 access
1796
1797 // h = X1 - zsqr (p256SubInternal: diff = y - x, so y=X1, x=zsqr)
1798 LDy(x1in)
1799 LDx(zsqrv)
1800 CALL p256SubInternal<>(SB)
1801 MOVV in+8(FP), a_ptr
1802
1803 // h2 = h * m_pre
1804 MOVV x0, y0
1805 MOVV x1, y1
1806 MOVV x2, y2
1807 MOVV x3, y3
1808 LDx(mv)
1809 CALL p256MulInternal<>(SB)
1810 STy(h2v) // save h2 before MulBy2 consumes y
1811
1812 // M = 3*h2 = 2*h2 + h2
1813 MOVV y0, x0
1814 MOVV y1, x1
1815 MOVV y2, x2
1816 MOVV y3, x3
1817 p256MulBy2Inline // x0..x3 = 2*h2
1818 LDy(h2v) // y0..y3 = original h2 (not overwritten)
1819 p256AddInline // x0..x3 = 2*h2 + h2 = 3*h2 = M
1820 STx(mv) // store M into mv
1821 MOVV in+8(FP), a_ptr // reload BEFORE next in1 dereference;
1822 // LDy(h2v) above re-clobbered y1==a_ptr
1823
1824 // s = (2*Y1)^2 = 4*Y1^2
1825 LDx(y1in)
1826 MOVV x0, y0
1827 MOVV x1, y1
1828 MOVV x2, y2
1829 MOVV x3, y3
1830 p256MulBy2Inline // x0..x3 = 2*Y1
1831 MOVV x0, y0
1832 MOVV x1, y1
1833 MOVV x2, y2
1834 MOVV x3, y3
1835 CALL p256SqrInternal<>(SB) // y0..y3 = (2*Y1)^2 = s
1836 STy(sv)
1837
1838 // yyyy = s^2 = 16*Y1^4
1839 MOVV y0, x0
1840 MOVV y1, x1
1841 MOVV y2, x2
1842 MOVV y3, x3
1843 MOVV in+8(FP), a_ptr
1844 CALL p256SqrInternal<>(SB) // y0..y3 = s^2 = 16*Y1^4
1845
1846 // 8*Y1^4 = yyyy / 2 (mod P halving)
1847 p256HalveInline // y0..y3 = yyyy/2
1848 STy(y3tmp) // stash; needed later for Y3
1849
1850 // S = X1 * s = 4*X1*Y1^2
1851 MOVV in+8(FP), a_ptr
1852 LDx(x1in)
1853 LDy(sv)
1854 CALL p256MulInternal<>(SB)
1855 STy(sv) // overwrite sv with S
1856
1857 // 2S = 8*X1*Y1^2
1858 MOVV y0, x0
1859 MOVV y1, x1
1860 MOVV y2, x2
1861 MOVV y3, x3
1862 p256MulBy2Inline
1863 STx(tmpv) // tmp = 2S
1864
1865 // Msqr = M^2
1866 LDx(mv)
1867 CALL p256SqrInternal<>(SB) // y0..y3 = M^2
1868
1869 // X3 = Msqr - 2S (diff = y - x, y = Msqr, x = 2S)
1870 LDx(tmpv)
1871 CALL p256SubInternal<>(SB)
1872 MOVV in+8(FP), a_ptr
1873 STx(x3out) // write res.x
1874
1875 // S - X3 (diff = y - x, y = S, x = X3)
1876 LDy(sv)
1877 CALL p256SubInternal<>(SB)
1878 MOVV in+8(FP), a_ptr
1879
1880 // M * (S - X3)
1881 MOVV x0, y0
1882 MOVV x1, y1
1883 MOVV x2, y2
1884 MOVV x3, y3
1885 LDx(mv)
1886 CALL p256MulInternal<>(SB)
1887
1888 // Y3 = M*(S-X3) - 8*Y1^4 (diff = y - x, y = previous result, x = y3tmp)
1889 LDx(y3tmp)
1890 CALL p256SubInternal<>(SB)
1891 MOVV in+8(FP), a_ptr
1892 STx(y3out) // write res.y
1893
1894 MOVV (8+32*6)(R3), R1 // restore return address
1895 RET
1896
1897 #undef y2in
1898 #undef x3out
1899 #undef y3out
1900 #undef z3out
1901 #define y2in(off) (off + 32)(b_ptr)
1902 #define x3out(off) (off)(b_ptr)
1903 #define y3out(off) (off + 32)(b_ptr)
1904 #define z3out(off) (off + 64)(b_ptr)
1905
1906 // p256PointAddAsm stack slots (addressed via the SP pseudo-register)
1907 #define z2sqrv(off) (8 + 32*0 + off)(R3)
1908 #define s1v(off) (8 + 32*1 + off)(R3)
1909 #define z1sqrv(off) (8 + 32*2 + off)(R3)
1910 #define rv2(off) (8 + 32*3 + off)(R3) // r
1911 #define u1v(off) (8 + 32*4 + off)(R3)
1912 #define u2v(off) (8 + 32*5 + off)(R3)
1913 #define hsqrv(off) (8 + 32*6 + off)(R3)
1914 #define rsqrv(off) (8 + 32*7 + off)(R3)
1915 #define hcubv(off) (8 + 32*8 + off)(R3)
1916 #define s2v2(off) (8 + 32*9 + off)(R3) // s2
1917 #define hv2(off) (8 + 32*10 + off)(R3) // h
1918 #define degflag(off) (8 + 32*11 + off)(R3) // degenerate-point flag (8 bytes used)
1919
1920 // y2inp is only valid while b_ptr still points at in2
1921 #define y2inp(off) (off+32)(b_ptr)
1922
1923 // func p256PointAddAsm(res, in1, in2 *P256Point) int
1924 TEXT ·p256PointAddAsm(SB),0,$392-32
1925 MOVV in1+8(FP), a_ptr
1926 MOVV in2+16(FP), b_ptr
1927
1928 // z2sqr = Z2^2
1929 LDx(z2in)
1930 CALL p256SqrInternal<>(SB)
1931 STy(z2sqrv)
1932
1933 // z2cub = z2sqr * Z2
1934 MOVV y0, x0
1935 MOVV y1, x1
1936 MOVV y2, x2
1937 MOVV y3, x3
1938 RELOAD_PTRS
1939 LDy(z2in)
1940 CALL p256MulInternal<>(SB)
1941
1942 // s1 = z2cub * Y1
1943 MOVV y0, x0
1944 MOVV y1, x1
1945 MOVV y2, x2
1946 MOVV y3, x3
1947 RELOAD_PTRS
1948 LDy(y1in)
1949 CALL p256MulInternal<>(SB)
1950 STy(s1v)
1951 RELOAD_PTRS
1952
1953 // z1sqr = Z1^2
1954 LDx(z1in)
1955 CALL p256SqrInternal<>(SB)
1956 STy(z1sqrv)
1957 RELOAD_PTRS
1958
1959 // z1cub = z1sqr * Z1
1960 LDx(z1sqrv)
1961 LDy(z1in)
1962 CALL p256MulInternal<>(SB)
1963
1964 // s2 = z1cub * Y2
1965 MOVV y0, x0
1966 MOVV y1, x1
1967 MOVV y2, x2
1968 MOVV y3, x3
1969 RELOAD_PTRS
1970 LDy(y2in)
1971 CALL p256MulInternal<>(SB)
1972
1973 // r = s2 - s1 (diff = y - x, y = s2, x = s1)
1974 LDx(s1v)
1975 CALL p256SubInternal<>(SB)
1976 STx(rv2)
1977
1978 // detect r == 0 mod P (r == 0 or r == P)
1979 MOVV $1, t2
1980 OR x0, x1, t0
1981 OR x2, x3, t1
1982 OR t1, t0, t0
1983 SGTU t0, R0, t1 // t1 = (t0 != 0) => 1 means nonzero
1984 XOR $1, t1, t1 // t1 = (t0 == 0) => 1 means r == 0
1985 MOVV t1, hlp1
1986
1987 XOR $-1, x0, t0
1988 MOVV p256const0<>(SB), t5
1989 XOR t5, x1, t1
1990 MOVV p256const1<>(SB), t6
1991 XOR t6, x3, t3
1992 OR t0, t1, t0
1993 OR x2, t3, t1
1994 OR t1, t0, t0
1995 SGTU t0, R0, t1
1996 XOR $1, t1, t1 // t1 = (r == P) => 1
1997 OR t1, hlp1, hlp1 // hlp1 = (r==0) OR (r==P)
1998
1999 MOVV hlp1, degflag(0) // spill immediately so later CALLs can't clobber it
2000
2001 // u1 = X1 * z2sqr
2002 RELOAD_PTRS // a_ptr/b_ptr were clobbered by previous CALL
2003 LDx(x1in)
2004 LDy(z2sqrv)
2005 CALL p256MulInternal<>(SB)
2006 STy(u1v)
2007 RELOAD_PTRS
2008
2009 // u2 = X2 * z1sqr
2010 LDx(x2in)
2011 LDy(z1sqrv)
2012 CALL p256MulInternal<>(SB)
2013
2014 // h = u2 - u1 (diff = y - x, y = u2, x = u1)
2015 LDx(u1v)
2016 CALL p256SubInternal<>(SB)
2017 STx(hv2)
2018
2019 // detect h == 0 mod P; AND into degflag (degenerate only if both are)
2020 MOVV $1, t2
2021 OR x0, x1, t0
2022 OR x2, x3, t1
2023 OR t1, t0, t0
2024 SGTU t0, R0, t1
2025 XOR $1, t1, t1
2026 AND t1, t2, hlp0
2027
2028 XOR $-1, x0, t0
2029 MOVV p256const0<>(SB), t5
2030 XOR t5, x1, t1
2031 MOVV p256const1<>(SB), t6
2032 XOR t6, x3, t3
2033 OR t0, t1, t0
2034 OR x2, t3, t1
2035 OR t1, t0, t0
2036 SGTU t0, R0, t1
2037 XOR $1, t1, t1
2038 OR t1, hlp0, hlp0 // hlp0 = (h==0) OR (h==P)
2039
2040 MOVV degflag(0), t4
2041 AND hlp0, t4, t4
2042 MOVV t4, degflag(0) // degflag = r-degenerate AND h-degenerate
2043
2044 // rsqr = r^2
2045 LDx(rv2)
2046 CALL p256SqrInternal<>(SB)
2047 STy(rsqrv)
2048 RELOAD_PTRS
2049
2050 // hsqr = h^2
2051 LDx(hv2)
2052 CALL p256SqrInternal<>(SB)
2053 STy(hsqrv)
2054 RELOAD_PTRS
2055
2056 // hcub = h * hsqr
2057 LDx(hsqrv)
2058 LDy(hv2)
2059 CALL p256MulInternal<>(SB)
2060 STy(hcubv)
2061 RELOAD_PTRS
2062
2063 // s2' = s1 * hcub
2064 LDx(s1v)
2065 LDy(hcubv)
2066 CALL p256MulInternal<>(SB)
2067 STy(s2v2)
2068 RELOAD_PTRS
2069
2070 // z3 = z1 * z2 * h
2071 LDx(z1in)
2072 LDy(z2in)
2073 CALL p256MulInternal<>(SB)
2074 MOVV y0, x0
2075 MOVV y1, x1
2076 MOVV y2, x2
2077 MOVV y3, x3
2078 RELOAD_PTRS
2079 LDy(hv2)
2080 CALL p256MulInternal<>(SB)
2081 MOVV res+0(FP), b_ptr // b_ptr now points at res; all reads
2082 // via the old b_ptr (in2) must be done
2083 STy(z3out)
2084
2085 // u2' = hsqr * u1
2086 LDx(hsqrv)
2087 LDy(u1v)
2088 CALL p256MulInternal<>(SB)
2089 STy(u2v)
2090
2091 // tmp = 2 * u2'
2092 p256MulBy2Inline
2093 RELOAD_PTRS
2094
2095 // x3_0 = rsqr - tmp (diff = y - x, y = rsqr, x = tmp)
2096 LDy(rsqrv)
2097 CALL p256SubInternal<>(SB)
2098 RELOAD_PTRS
2099 MOVV res+0(FP), b_ptr
2100
2101 // x3 = x3_0 - hcub
2102 MOVV x0, y0
2103 MOVV x1, y1
2104 MOVV x2, y2
2105 MOVV x3, y3
2106 LDx(hcubv)
2107 CALL p256SubInternal<>(SB)
2108 RELOAD_PTRS
2109 MOVV res+0(FP), b_ptr
2110 STx(x3out)
2111
2112 // tmp2 = u2' - x3
2113 LDy(u2v)
2114 CALL p256SubInternal<>(SB)
2115 RELOAD_PTRS
2116 MOVV res+0(FP), b_ptr
2117
2118 // tmp3 = r * tmp2
2119 LDy(rv2)
2120 CALL p256MulInternal<>(SB)
2121
2122 // y3 = tmp3 - s2' (diff = y - x, y = tmp3, x = s2')
2123 LDx(s2v2)
2124 CALL p256SubInternal<>(SB)
2125 RELOAD_PTRS
2126 MOVV res+0(FP), b_ptr
2127 STx(y3out)
2128
2129 MOVV degflag(0), t0
2130 MOVV t0, ret+24(FP)
2131
2132 RET
2133
View as plain text
