jmg
/
ed448goldilocks
1
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
174 Commits
2 Branches
1.6 MiB
 
 
 
 
 
Tree: 4d07cdf2ee
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '4d07cdf2ee'
${ noResults }
ed448goldilocks/src/p448/arch_neon_experimental/p448.c

723 lines
24 KiB
Raw Blame History 

  1. /* Copyright (c) 2014 Cryptography Research, Inc.

  2.  * Released under the MIT License.  See LICENSE.txt for license information.

  3.  */

  4. 

  5. #include "word.h"

  6. #include "p448.h"

  7. 

  8. static inline mask_t __attribute__((always_inline))

  9. is_zero (

  10.     word_t x

  11. ) {

  12.     dword_t xx = x;

  13.     xx--;

  14.     return xx >> WORD_BITS;

  15. }

  16. 

  17. static __inline__ uint64x2_t __attribute__((gnu_inline,always_inline,unused))

  18. xx_vaddup_u64(uint64x2_t x) {

  19.     __asm__ ("vadd.s64 %f0, %e0" : "+w"(x));

  20.     return x;

  21. }

  22. 

  23. static __inline__ int64x2_t __attribute__((gnu_inline,always_inline,unused))

  24. vrev128_s64(int64x2_t x) {

  25.     __asm__ ("vswp.s64 %e0, %f0" : "+w"(x));

  26.     return x;

  27. }

  28. 

  29. static __inline__ uint64x2_t __attribute__((gnu_inline,always_inline))

  30. vrev128_u64(uint64x2_t x) {

  31.     __asm__ ("vswp.s64 %e0, %f0" : "+w"(x));

  32.     return x;

  33. }

  34. 

  35. static inline void __attribute__((gnu_inline,always_inline,unused))

  36. smlal (

  37.     uint64_t *acc,

  38.     const uint32_t a,

  39.     const uint32_t b

  40. ) {

  41.     *acc += (int64_t)(int32_t)a * (int64_t)(int32_t)b;

  42. }

  43. 

  44. static inline void __attribute__((gnu_inline,always_inline,unused))

  45. smlal2 (

  46.     uint64_t *acc,

  47.     const uint32_t a,

  48.     const uint32_t b

  49. ) {

  50.     *acc += (int64_t)(int32_t)a * (int64_t)(int32_t)b * 2;

  51. }

  52. 

  53. static inline void __attribute__((gnu_inline,always_inline,unused))

  54. smull (

  55.     uint64_t *acc,

  56.     const uint32_t a,

  57.     const uint32_t b

  58. ) {

  59.     *acc = (int64_t)(int32_t)a * (int64_t)(int32_t)b;

  60. }

  61. 

  62. static inline void __attribute__((gnu_inline,always_inline,unused))

  63. smull2 (

  64.     uint64_t *acc,

  65.     const uint32_t a,

  66.     const uint32_t b

  67. ) {

  68.     *acc = (int64_t)(int32_t)a * (int64_t)(int32_t)b * 2;

  69. }

  70. 

  71. void

  72. p448_mul (

  73.     p448_t *__restrict__ cs,

  74.     const p448_t *as,

  75.     const p448_t *bs

  76. ) {

  77.     #define _bl0 "q0"

  78.     #define _bl0_0 "d0"

  79.     #define _bl0_1 "d1"

  80.     #define _bh0 "q1"

  81.     #define _bh0_0 "d2"

  82.     #define _bh0_1 "d3"

  83.     #define _bs0 "q2"

  84.     #define _bs0_0 "d4"

  85.     #define _bs0_1 "d5"

  86.     #define _bl2 "q3"

  87.     #define _bl2_0 "d6"

  88.     #define _bl2_1 "d7"

  89.     #define _bh2 "q4"

  90.     #define _bh2_0 "d8"

  91.     #define _bh2_1 "d9"

  92.     #define _bs2 "q5"

  93.     #define _bs2_0 "d10"

  94.     #define _bs2_1 "d11"

  95. 

  96.     #define _as0 "q6"

  97.     #define _as0_0 "d12"

  98.     #define _as0_1 "d13"

  99.     #define _as2 "q7"

  100.     #define _as2_0 "d14"

  101.     #define _as2_1 "d15"

  102.     #define _al0 "q8"

  103.     #define _al0_0 "d16"

  104.     #define _al0_1 "d17"

  105.     #define _ah0 "q9"

  106.     #define _ah0_0 "d18"

  107.     #define _ah0_1 "d19"

  108.     #define _al2 "q10"

  109.     #define _al2_0 "d20"

  110.     #define _al2_1 "d21"

  111.     #define _ah2 "q11"

  112.     #define _ah2_0 "d22"

  113.     #define _ah2_1 "d23"

  114. 

  115.     #define _a0a "q12"

  116.     #define _a0a_0 "d24"

  117.     #define _a0a_1 "d25"

  118.     #define _a0b "q13"

  119.     #define _a0b_0 "d26"

  120.     #define _a0b_1 "d27"

  121.     #define _a1a "q14"

  122.     #define _a1a_0 "d28"

  123.     #define _a1a_1 "d29"

  124.     #define _a1b "q15"

  125.     #define _a1b_0 "d30"

  126.     #define _a1b_1 "d31"

  127.     #define VMAC(op,result,a,b,n) #op" "result", "a", "b"[" #n "]\n\t"

  128.     #define VOP3(op,result,a,b)   #op" "result", "a", "b"\n\t"

  129.     #define VOP2(op,result,a)     #op" "result", "a"\n\t"

  130. 

  131.     int32x2_t *vc = (int32x2_t*) cs->limb;

  132. 

  133.     __asm__ __volatile__(

  134.         

  135.         "vld2.32 {"_al0_0","_al0_1","_ah0_0","_ah0_1"}, [%[a],:128]!" "\n\t"

  136.         VOP3(vadd.i32,_as0,_al0,_ah0)

  137.         

  138.         "vld2.32 {"_bl0_0","_bl0_1","_bh0_0","_bh0_1"}, [%[b],:128]!" "\n\t"

  139.         VOP3(vadd.i32,_bs0_1,_bl0_1,_bh0_1)

  140.         VOP3(vsub.i32,_bs0_0,_bl0_0,_bh0_0)

  141.             

  142.         "vld2.32 {"_bl2_0","_bl2_1","_bh2_0","_bh2_1"}, [%[b],:128]!" "\n\t"

  143.         VOP3(vadd.i32,_bs2,_bl2,_bh2)

  144.             

  145.         "vld2.32 {"_al2_0","_al2_1","_ah2_0","_ah2_1"}, [%[a],:128]!" "\n\t"

  146.         VOP3(vadd.i32,_as2,_al2,_ah2)

  147.         

  148.         VMAC(vmull.s32,_a0b,_as0_1,_bs2_1,0)

  149.         VMAC(vmlal.s32,_a0b,_as2_0,_bs2_0,0)

  150.         VMAC(vmlal.s32,_a0b,_as2_1,_bs0_1,0)

  151.         VMAC(vmlal.s32,_a0b,_as0_0,_bh0_0,0)

  152.             

  153.         VMAC(vmull.s32,_a1b,_as0_1,_bs2_1,1)

  154.         VMAC(vmlal.s32,_a1b,_as2_0,_bs2_0,1)

  155.         VMAC(vmlal.s32,_a1b,_as2_1,_bs0_1,1)

  156.         VMAC(vmlal.s32,_a1b,_as0_0,_bh0_0,1)

  157.             

  158.         VOP2(vmov,_a0a,_a0b)

  159.         VMAC(vmlal.s32,_a0a,_ah0_1,_bh2_1,0)

  160.         VMAC(vmlal.s32,_a0a,_ah2_0,_bh2_0,0)

  161.         VMAC(vmlal.s32,_a0a,_ah2_1,_bh0_1,0)

  162.         VMAC(vmlal.s32,_a0a,_ah0_0,_bl0_0,0)

  163.             

  164.         VMAC(vmlsl.s32,_a0b,_al0_1,_bl2_1,0)

  165.         VMAC(vmlsl.s32,_a0b,_al2_0,_bl2_0,0)

  166.         VMAC(vmlsl.s32,_a0b,_al2_1,_bl0_1,0)

  167.         VMAC(vmlal.s32,_a0b,_al0_0,_bs0_0,0)

  168.             

  169.         VOP2(vmov,_a1a,_a1b)

  170.         VMAC(vmlal.s32,_a1a,_ah0_1,_bh2_1,1)

  171.         VMAC(vmlal.s32,_a1a,_ah2_0,_bh2_0,1)

  172.         VMAC(vmlal.s32,_a1a,_ah2_1,_bh0_1,1)

  173.         VMAC(vmlal.s32,_a1a,_ah0_0,_bl0_0,1)

  174.             

  175.             VOP2(vswp,_a0b_1,_a0a_0)

  176.             

  177.         VMAC(vmlsl.s32,_a1b,_al0_1,_bl2_1,1)

  178.         VMAC(vmlsl.s32,_a1b,_al2_0,_bl2_0,1)

  179.         VMAC(vmlsl.s32,_a1b,_al2_1,_bl0_1,1)

  180.         VMAC(vmlal.s32,_a1b,_al0_0,_bs0_0,1)

  181.                 

  182.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  183.             VOP3(vsub.i32,_bs0_1,_bl0_1,_bh0_1)

  184.             VOP2(vmovn.i64,_a0b_0,_a0b)

  185.                 

  186.             VOP2(vswp,_a1b_1,_a1a_0)

  187.             VOP3(vadd.i64,_a1b,_a0a,_a1b)

  188.                     

  189.                     

  190.         VMAC(vmull.s32,_a0a,_as2_0,_bs2_1,0)

  191.             VOP2(vmovn.i64,_a0b_1,_a1b)

  192.         VMAC(vmlal.s32,_a0a,_as2_1,_bs2_0,0)

  193.             VOP3(vsra.u64,_a1a,_a1b,"#28")

  194.         VMAC(vmlal.s32,_a0a,_as0_0,_bh0_1,0)

  195.             VOP2(vbic.i32,_a0b,"#0xf0000000")

  196.         VMAC(vmlal.s32,_a0a,_as0_1,_bh0_0,0)

  197.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"

  198.                     

  199.         VMAC(vmull.s32,_a1b,_as2_0,_bs2_1,1)

  200.         VMAC(vmlal.s32,_a1b,_as2_1,_bs2_0,1)

  201.         VMAC(vmlal.s32,_a1b,_as0_0,_bh0_1,1)

  202.         VMAC(vmlal.s32,_a1b,_as0_1,_bh0_0,1)

  203. 

  204.         VOP2(vmov,_a0b_1,_a0a_1)

  205.         VOP3(vadd.i64,_a0b_0,_a0a_0,_a1a_0)

  206.         VOP3(vadd.i64,_a0a_0,_a0a_0,_a1a_1)

  207.         VMAC(vmlal.s32,_a0a,_ah2_0,_bh2_1,0)

  208.         VMAC(vmlal.s32,_a0a,_ah2_1,_bh2_0,0)

  209.         VMAC(vmlal.s32,_a0a,_ah0_0,_bl0_1,0)

  210.         VMAC(vmlal.s32,_a0a,_ah0_1,_bl0_0,0)

  211. 

  212.         VMAC(vmlsl.s32,_a0b,_al2_0,_bl2_1,0)

  213.         VMAC(vmlsl.s32,_a0b,_al2_1,_bl2_0,0)

  214.         VMAC(vmlal.s32,_a0b,_al0_0,_bs0_1,0)

  215.         VMAC(vmlal.s32,_a0b,_al0_1,_bs0_0,0)

  216. 

  217.         VOP2(vmov,_a1a,_a1b)

  218.         VMAC(vmlal.s32,_a1a,_ah2_0,_bh2_1,1)

  219.         VMAC(vmlal.s32,_a1a,_ah2_1,_bh2_0,1)

  220.         VMAC(vmlal.s32,_a1a,_ah0_0,_bl0_1,1)

  221.         VMAC(vmlal.s32,_a1a,_ah0_1,_bl0_0,1)

  222. 

  223.             VOP2(vswp,_a0b_1,_a0a_0)

  224. 

  225.         VMAC(vmlsl.s32,_a1b,_al2_0,_bl2_1,1)

  226.         VMAC(vmlsl.s32,_a1b,_al2_1,_bl2_0,1)

  227.         VMAC(vmlal.s32,_a1b,_al0_0,_bs0_1,1)

  228.         VMAC(vmlal.s32,_a1b,_al0_1,_bs0_0,1)

  229.                                         

  230.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  231.             VOP3(vsub.i32,_bs2_0,_bl2_0,_bh2_0)

  232.             VOP2(vmovn.i64,_a0b_0,_a0b)

  233.                         

  234.             VOP2(vswp,_a1b_1,_a1a_0)

  235.             VOP3(vadd.i64,_a1b,_a0a,_a1b)

  236. 

  237.         VMAC(vmull.s32,_a0a,_as2_1,_bs2_1,0)

  238.             VOP2(vmovn.i64,_a0b_1,_a1b)

  239.         VMAC(vmlal.s32,_a0a,_as0_0,_bh2_0,0)

  240.             VOP3(vsra.u64,_a1a,_a1b,"#28")

  241.         VMAC(vmlal.s32,_a0a,_as0_1,_bh0_1,0)

  242.             VOP2(vbic.i32,_a0b,"#0xf0000000")

  243.         VMAC(vmlal.s32,_a0a,_as2_0,_bh0_0,0)

  244.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"

  245. 

  246.         VMAC(vmull.s32,_a1b,_as2_1,_bs2_1,1)

  247.         VMAC(vmlal.s32,_a1b,_as0_0,_bh2_0,1)

  248.         VMAC(vmlal.s32,_a1b,_as0_1,_bh0_1,1)

  249.         VMAC(vmlal.s32,_a1b,_as2_0,_bh0_0,1)

  250. 

  251.         VOP2(vmov,_a0b_1,_a0a_1)

  252.         VOP3(vadd.i64,_a0b_0,_a0a_0,_a1a_0)

  253.         VOP3(vadd.i64,_a0a_0,_a0a_0,_a1a_1)

  254.         VMAC(vmlal.s32,_a0a,_ah2_1,_bh2_1,0)

  255.         VMAC(vmlal.s32,_a0a,_ah0_0,_bl2_0,0)

  256.         VMAC(vmlal.s32,_a0a,_ah0_1,_bl0_1,0)

  257.         VMAC(vmlal.s32,_a0a,_ah2_0,_bl0_0,0)

  258. 

  259.         VMAC(vmlsl.s32,_a0b,_al2_1,_bl2_1,0)

  260.         VMAC(vmlal.s32,_a0b,_al0_0,_bs2_0,0)

  261.         VMAC(vmlal.s32,_a0b,_al0_1,_bs0_1,0)

  262.         VMAC(vmlal.s32,_a0b,_al2_0,_bs0_0,0)

  263. 

  264.         VOP2(vmov,_a1a,_a1b)

  265.         VMAC(vmlal.s32,_a1a,_ah2_1,_bh2_1,1)

  266.         VMAC(vmlal.s32,_a1a,_ah0_0,_bl2_0,1)

  267.         VMAC(vmlal.s32,_a1a,_ah0_1,_bl0_1,1)

  268.         VMAC(vmlal.s32,_a1a,_ah2_0,_bl0_0,1)

  269. 

  270.             VOP2(vswp,_a0b_1,_a0a_0)

  271. 

  272.         VMAC(vmlsl.s32,_a1b,_al2_1,_bl2_1,1)

  273.         VMAC(vmlal.s32,_a1b,_al0_0,_bs2_0,1)

  274.         VMAC(vmlal.s32,_a1b,_al0_1,_bs0_1,1)

  275.         VMAC(vmlal.s32,_a1b,_al2_0,_bs0_0,1)

  276.                                                                 

  277.             VOP3(vsub.i32,_bs2_1,_bl2_1,_bh2_1)

  278.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  279.             VOP2(vmovn.i64,_a0b_0,_a0b)

  280.                         

  281.             VOP2(vswp,_a1b_1,_a1a_0)

  282.             VOP3(vadd.i64,_a1b,_a0a,_a1b)

  283. 

  284.         VMAC(vmull.s32,_a0a,_as0_0,_bh2_1,0)

  285.             VOP2(vmovn.i64,_a0b_1,_a1b)

  286.         VMAC(vmlal.s32,_a0a,_as0_1,_bh2_0,0)

  287.             VOP3(vsra.u64,_a1a,_a1b,"#28")

  288.         VMAC(vmlal.s32,_a0a,_as2_0,_bh0_1,0)

  289.             VOP2(vbic.i32,_a0b,"#0xf0000000")

  290.         VMAC(vmlal.s32,_a0a,_as2_1,_bh0_0,0)

  291.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"

  292. 

  293.         VMAC(vmull.s32,_a1b,_as0_0,_bh2_1,1)

  294.         VMAC(vmlal.s32,_a1b,_as0_1,_bh2_0,1)

  295.         VMAC(vmlal.s32,_a1b,_as2_0,_bh0_1,1)

  296.         VMAC(vmlal.s32,_a1b,_as2_1,_bh0_0,1)

  297. 

  298.         VOP2(vmov,_a0b_1,_a0a_1)

  299.         VOP3(vadd.i64,_a0b_0,_a0a_0,_a1a_0)

  300.         VOP3(vadd.i64,_a0a_0,_a0a_0,_a1a_1)

  301.         VMAC(vmlal.s32,_a0a,_ah0_0,_bl2_1,0)

  302.         VMAC(vmlal.s32,_a0a,_ah0_1,_bl2_0,0)

  303.         VMAC(vmlal.s32,_a0a,_ah2_0,_bl0_1,0)

  304.         VMAC(vmlal.s32,_a0a,_ah2_1,_bl0_0,0)

  305. 

  306.         VMAC(vmlal.s32,_a0b,_al0_0,_bs2_1,0)

  307.         VMAC(vmlal.s32,_a0b,_al0_1,_bs2_0,0)

  308.         VMAC(vmlal.s32,_a0b,_al2_0,_bs0_1,0)

  309.         VMAC(vmlal.s32,_a0b,_al2_1,_bs0_0,0)

  310. 

  311.         VOP2(vmov,_a1a,_a1b)

  312.         VMAC(vmlal.s32,_a1a,_ah0_0,_bl2_1,1)

  313.         VMAC(vmlal.s32,_a1a,_ah0_1,_bl2_0,1)

  314.         VMAC(vmlal.s32,_a1a,_ah2_0,_bl0_1,1)

  315.         VMAC(vmlal.s32,_a1a,_ah2_1,_bl0_0,1)

  316. 

  317.             VOP2(vswp,_a0b_1,_a0a_0)

  318. 

  319.         VMAC(vmlal.s32,_a1b,_al0_0,_bs2_1,1)

  320.         VMAC(vmlal.s32,_a1b,_al0_1,_bs2_0,1)

  321.         VMAC(vmlal.s32,_a1b,_al2_0,_bs0_1,1)

  322.         VMAC(vmlal.s32,_a1b,_al2_1,_bs0_0,1)

  323.                         

  324.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  325.             VOP2(vmovn.i64,_a0b_0,_a0b)

  326.                                                                                             

  327.             VOP2(vswp,_a1b_1,_a1a_0)

  328.             VOP3(vadd.i64,_a0a,_a0a,_a1b)

  329. 

  330.             VOP2(vmovn.i64,_a0b_1,_a0a)

  331.             VOP3(vsra.u64,_a1a,_a0a,"#28")

  332.                                                                                             

  333.             VOP2(vbic.i32,_a0b,"#0xf0000000") 

  334.                                                                                             

  335.         VOP2(vswp,_a1a_0,_a1a_1)

  336.                                                                                             

  337.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"  

  338.             "sub %[c], #64" "\n\t"

  339.                                                                                                 

  340.         VOP3(vadd.i64,_a1a_1,_a1a_1,_a1a_0)

  341.         

  342.             "vldmia %[c], {"_a0a_0", "_a0a_1", "_a0b_0"}" "\n\t"

  343.             VOP2(vaddw.s32,_a1a,_a0a_0)

  344.             VOP2(vmovn.i64,_a0a_0,_a1a)

  345.             VOP2(vshr.s64,_a1a,"#28")

  346.                                                 

  347.             VOP2(vaddw.s32,_a1a,_a0a_1)

  348.             VOP2(vmovn.i64,_a0a_1,_a1a)

  349.             VOP2(vshr.s64,_a1a,"#28")

  350.                                                                                                     

  351.             VOP2(vbic.i32,_a0a,"#0xf0000000")

  352.                                                 

  353.             VOP2(vaddw.s32,_a1a,_a0b_0) 

  354.             VOP2(vmovn.i64,_a0b_0,_a1a)

  355.             

  356.             "vstmia %[c], {"_a0a_0", "_a0a_1", "_a0b_0"}" "\n\t"

  357.         

  358.         : [a]"+r"(as)

  359.         , [b]"+r"(bs)

  360.         , [c]"+r"(vc)

  361.                             

  362.         :: "q0","q1","q2","q3",

  363.             "q4","q5","q6","q7",

  364.             "q8","q9","q10","q11",

  365.             "q12","q13","q14","q15",

  366.             "memory"

  367.     );

  368. }

  369. 

  370. void

  371. p448_sqr (

  372.     p448_t *__restrict__ cs,

  373.     const p448_t *bs

  374. ) {

  375.     int32x2_t *vc = (int32x2_t*) cs->limb;

  376. 

  377.     __asm__ __volatile__ (

  378.         "vld2.32 {"_bl0_0","_bl0_1","_bh0_0","_bh0_1"}, [%[b],:128]!" "\n\t"

  379.         VOP3(vadd.i32,_bs0_1,_bl0_1,_bh0_1) /* 0 .. 2^30 */

  380.         VOP3(vsub.i32,_bs0_0,_bl0_0,_bh0_0) /* +- 2^29 */

  381.         VOP3(vadd.i32,_as0,_bl0,_bh0)       /* 0 .. 2^30 */

  382.             

  383.         "vld2.32 {"_bl2_0","_bl2_1","_bh2_0","_bh2_1"}, [%[b],:128]!" "\n\t"

  384.         VOP3(vadd.i32,_bs2,_bl2,_bh2)       /* 0 .. 2^30 */

  385.         VOP2(vmov,_as2,_bs2)

  386.         

  387.         VMAC(vqdmull.s32,_a0b,_as0_1,_bs2_1,0) /* 0 .. 8 * 2^58.  danger for vqdmlal is 32 */

  388.         VMAC(vmlal.s32,_a0b,_as2_0,_bs2_0,0)   /* 0 .. 12 */

  389.         VMAC(vmlal.s32,_a0b,_as0_0,_bh0_0,0)   /* 0 .. 14 */

  390.             

  391.         VMAC(vqdmull.s32,_a1b,_as0_1,_bs2_1,1) /* 0 .. 8 */

  392.         VMAC(vmlal.s32,_a1b,_as2_0,_bs2_0,1)   /* 0 .. 14 */

  393.         VMAC(vmlal.s32,_a1b,_as0_0,_bh0_0,1)   /* 0 .. 16 */

  394.             

  395.         VOP2(vmov,_a0a,_a0b)                   /* 0 .. 14 */

  396.         VMAC(vqdmlal.s32,_a0a,_bh0_1,_bh2_1,0) /* 0 .. 16 */

  397.         VMAC(vmlal.s32,_a0a,_bh2_0,_bh2_0,0)   /* 0 .. 17 */

  398.         VMAC(vmlal.s32,_a0a,_bh0_0,_bl0_0,0)   /* 0 .. 18 */

  399.             

  400.         VMAC(vqdmlsl.s32,_a0b,_bl0_1,_bl2_1,0) /*-2 .. 14 */

  401.         VMAC(vmlsl.s32,_a0b,_bl2_0,_bl2_0,0)   /*-3 .. 14 */

  402.         VMAC(vmlal.s32,_a0b,_bl0_0,_bs0_0,0)   /*-4 .. 15 */

  403.             

  404.         VOP2(vmov,_a1a,_a1b)

  405.         VMAC(vqdmlal.s32,_a1a,_bh0_1,_bh2_1,1) /* 0 .. 18 */

  406.         VMAC(vmlal.s32,_a1a,_bh2_0,_bh2_0,1)   /* 0 .. 19 */

  407.         VMAC(vmlal.s32,_a1a,_bh0_0,_bl0_0,1)   /* 0 .. 20 */

  408.             

  409.             VOP2(vswp,_a0b_1,_a0a_0)

  410.             

  411.         VMAC(vqdmlsl.s32,_a1b,_bl0_1,_bl2_1,1) /*-2 .. 16 */

  412.         VMAC(vmlsl.s32,_a1b,_bl2_0,_bl2_0,1)   /*-3 .. 16 */

  413.         VMAC(vmlal.s32,_a1b,_bl0_0,_bs0_0,1)   /*-4 .. 17 */

  414.                 

  415.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  416.             VOP3(vsub.i32,_bs0_1,_bl0_1,_bh0_1)

  417.             VOP2(vmovn.i64,_a0b_0,_a0b)

  418.                 

  419.             VOP2(vswp,_a1b_1,_a1a_0)

  420.             VOP3(vadd.i64,_a1b,_a0a,_a1b)

  421.                     

  422.                     

  423.         VMAC(vqdmull.s32,_a0a,_as2_0,_bs2_1,0) /* 0 .. 8 */

  424.             VOP2(vmovn.i64,_a0b_1,_a1b)

  425.             VOP3(vsra.u64,_a1a,_a1b,"#28")

  426.         VMAC(vqdmlal.s32,_a0a,_as0_0,_bh0_1,0) /* 0 .. 12 */

  427.             VOP2(vbic.i32,_a0b,"#0xf0000000")

  428.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"

  429.                     

  430.         VMAC(vqdmull.s32,_a1b,_as2_0,_bs2_1,1) /* 0 .. 8 */

  431.         VMAC(vqdmlal.s32,_a1b,_as0_0,_bh0_1,1) /* 0 .. 12 */

  432. 

  433.         VOP2(vmov,_a0b,_a0a)               /* 0 .. 12 */

  434.         VMAC(vqdmlal.s32,_a0a,_bh2_0,_bh2_1,0) /* 0 .. 14 */

  435.         VMAC(vqdmlal.s32,_a0a,_bh0_0,_bl0_1,0) /* 0 .. 16 */

  436. 

  437.         VMAC(vqdmlsl.s32,_a0b,_bl2_0,_bl2_1,0) /*-2 .. 12 */

  438.         VMAC(vqdmlal.s32,_a0b,_bl0_0,_bs0_1,0) /*-4 .. 14 */

  439.         VOP3(vadd.i64,_a0a_0,_a0a_0,_a1a_1)

  440.         VOP3(vadd.i64,_a0b_0,_a0b_0,_a1a_0)

  441. 

  442.         VOP2(vmov,_a1a,_a1b)                   /* 0 .. 12 */

  443.         VMAC(vqdmlal.s32,_a1a,_bh2_0,_bh2_1,1) /* 0 .. 14 */

  444.         VMAC(vqdmlal.s32,_a1a,_bh0_0,_bl0_1,1) /* 0 .. 16 */

  445. 

  446.             VOP2(vswp,_a0b_1,_a0a_0)

  447. 

  448.         VMAC(vqdmlsl.s32,_a1b,_bl2_0,_bl2_1,1) /*-2 .. 12 */

  449.         VMAC(vqdmlal.s32,_a1b,_bl0_0,_bs0_1,1) /*-4 .. 14 */

  450.                                         

  451.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  452.             VOP3(vsub.i32,_bs2_0,_bl2_0,_bh2_0)

  453.             VOP2(vmovn.i64,_a0b_0,_a0b)

  454.                         

  455.             VOP2(vswp,_a1b_1,_a1a_0)

  456.             VOP3(vadd.i64,_a1b,_a0a,_a1b)

  457. 

  458.         VMAC(vmull.s32,_a0a,_as2_1,_bs2_1,0)

  459.             VOP2(vmovn.i64,_a0b_1,_a1b)

  460.         VMAC(vqdmlal.s32,_a0a,_as0_0,_bh2_0,0)

  461.             VOP3(vsra.u64,_a1a,_a1b,"#28")

  462.         VMAC(vmlal.s32,_a0a,_as0_1,_bh0_1,0)

  463.             VOP2(vbic.i32,_a0b,"#0xf0000000")

  464.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"

  465. 

  466.         VMAC(vmull.s32,_a1b,_as2_1,_bs2_1,1)

  467.         VMAC(vqdmlal.s32,_a1b,_as0_0,_bh2_0,1)

  468.         VMAC(vmlal.s32,_a1b,_as0_1,_bh0_1,1)

  469. 

  470.         VOP2(vmov,_a0b_1,_a0a_1)

  471.         VOP3(vadd.i64,_a0b_0,_a0a_0,_a1a_0)

  472.         VOP3(vadd.i64,_a0a_0,_a0a_0,_a1a_1)

  473.         VMAC(vmlal.s32,_a0a,_bh2_1,_bh2_1,0)

  474.         VMAC(vqdmlal.s32,_a0a,_bh0_0,_bl2_0,0)

  475.         VMAC(vmlal.s32,_a0a,_bh0_1,_bl0_1,0)

  476. 

  477.         VMAC(vmlsl.s32,_a0b,_bl2_1,_bl2_1,0)

  478.         VMAC(vqdmlal.s32,_a0b,_bl0_0,_bs2_0,0)

  479.         VMAC(vmlal.s32,_a0b,_bl0_1,_bs0_1,0)

  480. 

  481.         VOP2(vmov,_a1a,_a1b)

  482.         VMAC(vmlal.s32,_a1a,_bh2_1,_bh2_1,1)

  483.         VMAC(vqdmlal.s32,_a1a,_bh0_0,_bl2_0,1)

  484.         VMAC(vmlal.s32,_a1a,_bh0_1,_bl0_1,1)

  485. 

  486.             VOP2(vswp,_a0b_1,_a0a_0)

  487. 

  488.         VMAC(vmlsl.s32,_a1b,_bl2_1,_bl2_1,1)

  489.         VMAC(vqdmlal.s32,_a1b,_bl0_0,_bs2_0,1)

  490.         VMAC(vmlal.s32,_a1b,_bl0_1,_bs0_1,1)

  491.                                                                 

  492.             VOP3(vsub.i32,_bs2_1,_bl2_1,_bh2_1)

  493.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  494.             VOP2(vmovn.i64,_a0b_0,_a0b)

  495.                         

  496.             VOP2(vswp,_a1b_1,_a1a_0)

  497.             VOP3(vadd.i64,_a1b,_a0a,_a1b)

  498. 

  499.         VMAC(vqdmull.s32,_a0a,_as0_0,_bh2_1,0)

  500.             VOP2(vmovn.i64,_a0b_1,_a1b)

  501.             VOP3(vsra.u64,_a1a,_a1b,"#28")

  502.         VMAC(vqdmlal.s32,_a0a,_as2_0,_bh0_1,0)

  503.             VOP2(vbic.i32,_a0b,"#0xf0000000")

  504.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"

  505. 

  506.         VMAC(vqdmull.s32,_a1b,_as0_0,_bh2_1,1)

  507.         VMAC(vqdmlal.s32,_a1b,_as2_0,_bh0_1,1)

  508. 

  509.         VOP2(vmov,_a0b_1,_a0a_1)

  510.         VOP3(vadd.i64,_a0b_0,_a0a_0,_a1a_0)

  511.         VOP3(vadd.i64,_a0a_0,_a0a_0,_a1a_1)

  512.         VMAC(vqdmlal.s32,_a0a,_bh0_0,_bl2_1,0)

  513.         VMAC(vqdmlal.s32,_a0a,_bh2_0,_bl0_1,0)

  514. 

  515.         VMAC(vqdmlal.s32,_a0b,_bl0_0,_bs2_1,0)

  516.         VMAC(vqdmlal.s32,_a0b,_bl2_0,_bs0_1,0)

  517. 

  518.         VOP2(vmov,_a1a,_a1b)

  519.         VMAC(vqdmlal.s32,_a1a,_bh0_0,_bl2_1,1)

  520.         VMAC(vqdmlal.s32,_a1a,_bh2_0,_bl0_1,1)

  521. 

  522.             VOP2(vswp,_a0b_1,_a0a_0)

  523. 

  524.         VMAC(vqdmlal.s32,_a1b,_bl0_0,_bs2_1,1)

  525.         VMAC(vqdmlal.s32,_a1b,_bl2_0,_bs0_1,1)

  526.                         

  527.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  528.             VOP2(vmovn.i64,_a0b_0,_a0b)

  529.                                                                                             

  530.             VOP2(vswp,_a1b_1,_a1a_0)

  531.             VOP3(vadd.i64,_a0a,_a0a,_a1b)

  532. 

  533.             VOP2(vmovn.i64,_a0b_1,_a0a)

  534.             VOP3(vsra.u64,_a1a,_a0a,"#28")

  535.                                                                                             

  536.             VOP2(vbic.i32,_a0b,"#0xf0000000") 

  537.                                                                                             

  538.         VOP2(vswp,_a1a_0,_a1a_1)

  539.                                                                                             

  540.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"  

  541.             "sub %[c], #64" "\n\t"

  542.                                                                                                 

  543.         VOP3(vadd.i64,_a1a_1,_a1a_1,_a1a_0)

  544.         

  545.             "vldmia %[c], {"_a0a_0", "_a0a_1", "_a0b_0"}" "\n\t"

  546.             VOP2(vaddw.s32,_a1a,_a0a_0)

  547.             VOP2(vmovn.i64,_a0a_0,_a1a)

  548.             VOP2(vshr.s64,_a1a,"#28")

  549.                                                 

  550.             VOP2(vaddw.s32,_a1a,_a0a_1)

  551.             VOP2(vmovn.i64,_a0a_1,_a1a)

  552.             VOP2(vshr.s64,_a1a,"#28")

  553.                                                                                                     

  554.             VOP2(vbic.i32,_a0a,"#0xf0000000")

  555.                                                 

  556.             VOP2(vaddw.s32,_a1a,_a0b_0) 

  557.             VOP2(vmovn.i64,_a0b_0,_a1a)

  558.             

  559.             "vstmia %[c], {"_a0a_0", "_a0a_1", "_a0b_0"}" "\n\t"

  560.         

  561.         : [b]"+r"(bs)

  562.         , [c]"+r"(vc)

  563.                             

  564.         :: "q0","q1","q2","q3",

  565.             "q4","q5","q6","q7",

  566.             "q12","q13","q14","q15",

  567.             "memory"

  568.     );

  569. }

  570. 

  571. void

  572. p448_mulw (

  573.     p448_t *__restrict__ cs,

  574.     const p448_t *as,

  575.     uint64_t b

  576. ) { 

  577.     uint32x2_t vmask = {(1<<28) - 1, (1<<28)-1};

  578.     

  579.     uint64x2_t accum;

  580.     const uint32x2_t *va = (const uint32x2_t *) as->limb;

  581.     uint32x2_t *vo = (uint32x2_t *) cs->limb;

  582.     uint32x2_t vc, vn;

  583.     uint32x2_t vb = {b & ((1<<28)-1), b>>28};

  584.     

  585.     accum = vmull_lane_u32(va[7], vb, 1);

  586.     accum = xx_vaddup_u64(vrev128_u64(accum));

  587.     

  588.     vc = va[0];

  589.     accum = vmlal_lane_u32(accum, vc, vb, 0);

  590.     vo[0] = vmovn_u64(accum) & vmask;

  591.     accum = vshrq_n_u64(accum,28);

  592.     

  593.     /* PERF: the right way to do this is to reduce behind, i.e.

  594.      * vmull + vmlal round 0

  595.      * vmull + vmlal round 1

  596.      * vmull + vmlal round 2

  597.      * vsraq round 0, 1

  598.      * vmull + vmlal round 3

  599.      * vsraq round 1, 2

  600.      * ...

  601.      */

  602.     

  603.     int i;

  604.     for (i=1; i<8; i++) {

  605.         vn = va[i];

  606.         accum = vmlal_lane_u32(accum, vc, vb, 1);

  607.         accum = vmlal_lane_u32(accum, vn, vb, 0);

  608.         vo[i] = vmovn_u64(accum) & vmask;

  609.         accum = vshrq_n_u64(accum,28);

  610.         vc = vn;

  611.     }

  612.         

  613.     accum = xx_vaddup_u64(vrev128_u64(accum));

  614.     accum = vaddw_u32(accum, vo[0]);

  615.     vo[0] = vmovn_u64(accum) & vmask;

  616.     

  617.     accum = vshrq_n_u64(accum,28);

  618.     vo[1] += vmovn_u64(accum);

  619. }

  620. 

  621. /* TODO: vectorize? */

  622. void

  623. p448_strong_reduce (

  624.     p448_t *a

  625. ) { 

  626.     word_t mask = (1ull<<28)-1;

  627. 

  628.     /* first, clear high */

  629.     a->limb[1] += a->limb[15]>>28;

  630.     a->limb[0] += a->limb[15]>>28;

  631.     a->limb[15] &= mask;

  632. 

  633.     /* now the total is less than 2^448 - 2^(448-56) + 2^(448-56+8) < 2p */

  634. 

  635.     /* compute total_value - p.  No need to reduce mod p. */

  636. 

  637.     dsword_t scarry = 0;

  638.     int i;

  639.     for (i=0; i<16; i++) {

  640.         scarry = scarry + a->limb[LIMBPERM(i)] - ((i==8)?mask-1:mask);

  641.         a->limb[LIMBPERM(i)] = scarry & mask;

  642.         scarry >>= 28;

  643.     }

  644. 

  645.     /* uncommon case: it was >= p, so now scarry = 0 and this = x

  646.     * common case: it was < p, so now scarry = -1 and this = x - p + 2^448

  647.     * so let's add back in p.  will carry back off the top for 2^448.

  648.     */

  649. 

  650.     assert(is_zero(scarry) | is_zero(scarry+1));

  651. 

  652.     word_t scarry_mask = scarry & mask;

  653.     dword_t carry = 0;

  654. 

  655.     /* add it back */

  656.     for (i=0; i<16; i++) {

  657.         carry = carry + a->limb[LIMBPERM(i)] + ((i==8)?(scarry_mask&~1):scarry_mask);

  658.         a->limb[LIMBPERM(i)] = carry & mask;

  659.         carry >>= 28;

  660.     }

  661. 

  662.     assert(is_zero(carry + scarry));

  663. }

  664. 

  665. void

  666. p448_serialize (

  667.     uint8_t *serial,

  668.     const struct p448_t *x

  669. ) {

  670.     int i,j;

  671.     p448_t red;

  672.     p448_copy(&red, x);

  673.     p448_strong_reduce(&red);

  674.     

  675.     for (i=0; i<8; i++) {

  676.         uint64_t limb = red.limb[LIMBPERM(2*i)] + (((uint64_t)red.limb[LIMBPERM(2*i+1)])<<28);

  677.         for (j=0; j<7; j++) {

  678.             serial[7*i+j] = limb;

  679.             limb >>= 8;

  680.         }

  681.         assert(limb == 0);

  682.     }

  683. }

  684. 

  685. mask_t

  686. p448_deserialize (

  687.     p448_t *x,

  688.     const uint8_t serial[56]

  689. ) {

  690.     int i,j;

  691.     for (i=0; i<8; i++) {

  692.         uint64_t out = 0;

  693.         for (j=0; j<7; j++) {

  694.             out |= ((uint64_t)serial[7*i+j])<<(8*j);

  695.         }

  696.         x->limb[LIMBPERM(2*i)] = out & ((1ull<<28)-1);

  697.         x->limb[LIMBPERM(2*i+1)] = out >> 28;

  698.     }

  699.     

  700.     /* Check for reduction.

  701.      *

  702.      * The idea is to create a variable ge which is all ones (rather, 56 ones)

  703.      * if and only if the low $i$ words of $x$ are >= those of p.

  704.      *

  705.      * Remember p = little_endian(1111,1111,1111,1111,1110,1111,1111,1111)

  706.      */

  707.     uint32_t ge = -1, mask = (1ull<<28)-1;

  708.     for (i=0; i<8; i++) {

  709.         ge &= x->limb[LIMBPERM(i)];

  710.     }

  711.     

  712.     /* At this point, ge = 1111 iff bottom are all 1111.  Now propagate if 1110, or set if 1111 */

  713.     ge = (ge & (x->limb[LIMBPERM(8)] + 1)) | is_zero(x->limb[LIMBPERM(8)] ^ mask);

  714.     

  715.     /* Propagate the rest */

  716.     for (i=9; i<16; i++) {

  717.         ge &= x->limb[LIMBPERM(i)];

  718.     }

  719.     

  720.     return ~is_zero(ge ^ mask);

  721. }