jmg
/
ed448goldilocks
1
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
352 Commits
2 Branches
1.6 MiB
 
 
 
 
 
Tree: a585d7f148
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'a585d7f148'
${ noResults }
ed448goldilocks/src/p448/arch_neon/f_impl.c

593 lines
21 KiB
Raw Blame History 

  1. /* Copyright (c) 2014 Cryptography Research, Inc.

  2.  * Released under the MIT License.  See LICENSE.txt for license information.

  3.  */

  4. 

  5. #include "f_field.h"

  6. 

  7. static __inline__ uint64x2_t __attribute__((gnu_inline,always_inline,unused))

  8. xx_vaddup_u64(uint64x2_t x) {

  9.     __asm__ ("vadd.s64 %f0, %e0" : "+w"(x));

  10.     return x;

  11. }

  12. 

  13. static __inline__ int64x2_t __attribute__((gnu_inline,always_inline,unused))

  14. vrev128_s64(int64x2_t x) {

  15.     __asm__ ("vswp.s64 %e0, %f0" : "+w"(x));

  16.     return x;

  17. }

  18. 

  19. static __inline__ uint64x2_t __attribute__((gnu_inline,always_inline))

  20. vrev128_u64(uint64x2_t x) {

  21.     __asm__ ("vswp.s64 %e0, %f0" : "+w"(x));

  22.     return x;

  23. }

  24. 

  25. static inline void __attribute__((gnu_inline,always_inline,unused))

  26. smlal (

  27.     uint64_t *acc,

  28.     const uint32_t a,

  29.     const uint32_t b

  30. ) {

  31.     *acc += (int64_t)(int32_t)a * (int64_t)(int32_t)b;

  32. }

  33. 

  34. static inline void __attribute__((gnu_inline,always_inline,unused))

  35. smlal2 (

  36.     uint64_t *acc,

  37.     const uint32_t a,

  38.     const uint32_t b

  39. ) {

  40.     *acc += (int64_t)(int32_t)a * (int64_t)(int32_t)b * 2;

  41. }

  42. 

  43. static inline void __attribute__((gnu_inline,always_inline,unused))

  44. smull (

  45.     uint64_t *acc,

  46.     const uint32_t a,

  47.     const uint32_t b

  48. ) {

  49.     *acc = (int64_t)(int32_t)a * (int64_t)(int32_t)b;

  50. }

  51. 

  52. static inline void __attribute__((gnu_inline,always_inline,unused))

  53. smull2 (

  54.     uint64_t *acc,

  55.     const uint32_t a,

  56.     const uint32_t b

  57. ) {

  58.     *acc = (int64_t)(int32_t)a * (int64_t)(int32_t)b * 2;

  59. }

  60. 

  61. void gf_mul (gf_s *__restrict__ cs, const gf as, const gf bs) {

  62.     #define _bl0 "q0"

  63.     #define _bl0_0 "d0"

  64.     #define _bl0_1 "d1"

  65.     #define _bh0 "q1"

  66.     #define _bh0_0 "d2"

  67.     #define _bh0_1 "d3"

  68.     #define _bs0 "q2"

  69.     #define _bs0_0 "d4"

  70.     #define _bs0_1 "d5"

  71.     #define _bl2 "q3"

  72.     #define _bl2_0 "d6"

  73.     #define _bl2_1 "d7"

  74.     #define _bh2 "q4"

  75.     #define _bh2_0 "d8"

  76.     #define _bh2_1 "d9"

  77.     #define _bs2 "q5"

  78.     #define _bs2_0 "d10"

  79.     #define _bs2_1 "d11"

  80. 

  81.     #define _as0 "q6"

  82.     #define _as0_0 "d12"

  83.     #define _as0_1 "d13"

  84.     #define _as2 "q7"

  85.     #define _as2_0 "d14"

  86.     #define _as2_1 "d15"

  87.     #define _al0 "q8"

  88.     #define _al0_0 "d16"

  89.     #define _al0_1 "d17"

  90.     #define _ah0 "q9"

  91.     #define _ah0_0 "d18"

  92.     #define _ah0_1 "d19"

  93.     #define _al2 "q10"

  94.     #define _al2_0 "d20"

  95.     #define _al2_1 "d21"

  96.     #define _ah2 "q11"

  97.     #define _ah2_0 "d22"

  98.     #define _ah2_1 "d23"

  99. 

  100.     #define _a0a "q12"

  101.     #define _a0a_0 "d24"

  102.     #define _a0a_1 "d25"

  103.     #define _a0b "q13"

  104.     #define _a0b_0 "d26"

  105.     #define _a0b_1 "d27"

  106.     #define _a1a "q14"

  107.     #define _a1a_0 "d28"

  108.     #define _a1a_1 "d29"

  109.     #define _a1b "q15"

  110.     #define _a1b_0 "d30"

  111.     #define _a1b_1 "d31"

  112.     #define VMAC(op,result,a,b,n) #op" "result", "a", "b"[" #n "]\n\t"

  113.     #define VOP3(op,result,a,b)   #op" "result", "a", "b"\n\t"

  114.     #define VOP2(op,result,a)     #op" "result", "a"\n\t"

  115. 

  116.     int32x2_t *vc = (int32x2_t*) cs->limb;

  117. 

  118.     __asm__ __volatile__(

  119.         

  120.         "vld2.32 {"_al0_0","_al0_1","_ah0_0","_ah0_1"}, [%[a],:128]!" "\n\t"

  121.         VOP3(vadd.i32,_as0,_al0,_ah0)

  122.         

  123.         "vld2.32 {"_bl0_0","_bl0_1","_bh0_0","_bh0_1"}, [%[b],:128]!" "\n\t"

  124.         VOP3(vadd.i32,_bs0_1,_bl0_1,_bh0_1)

  125.         VOP3(vsub.i32,_bs0_0,_bl0_0,_bh0_0)

  126.             

  127.         "vld2.32 {"_bl2_0","_bl2_1","_bh2_0","_bh2_1"}, [%[b],:128]!" "\n\t"

  128.         VOP3(vadd.i32,_bs2,_bl2,_bh2)

  129.             

  130.         "vld2.32 {"_al2_0","_al2_1","_ah2_0","_ah2_1"}, [%[a],:128]!" "\n\t"

  131.         VOP3(vadd.i32,_as2,_al2,_ah2)

  132.         

  133.         VMAC(vmull.s32,_a0b,_as0_1,_bs2_1,0)

  134.         VMAC(vmlal.s32,_a0b,_as2_0,_bs2_0,0)

  135.         VMAC(vmlal.s32,_a0b,_as2_1,_bs0_1,0)

  136.         VMAC(vmlal.s32,_a0b,_as0_0,_bh0_0,0)

  137.             

  138.         VMAC(vmull.s32,_a1b,_as0_1,_bs2_1,1)

  139.         VMAC(vmlal.s32,_a1b,_as2_0,_bs2_0,1)

  140.         VMAC(vmlal.s32,_a1b,_as2_1,_bs0_1,1)

  141.         VMAC(vmlal.s32,_a1b,_as0_0,_bh0_0,1)

  142.             

  143.         VOP2(vmov,_a0a,_a0b)

  144.         VMAC(vmlal.s32,_a0a,_ah0_1,_bh2_1,0)

  145.         VMAC(vmlal.s32,_a0a,_ah2_0,_bh2_0,0)

  146.         VMAC(vmlal.s32,_a0a,_ah2_1,_bh0_1,0)

  147.         VMAC(vmlal.s32,_a0a,_ah0_0,_bl0_0,0)

  148.             

  149.         VMAC(vmlsl.s32,_a0b,_al0_1,_bl2_1,0)

  150.         VMAC(vmlsl.s32,_a0b,_al2_0,_bl2_0,0)

  151.         VMAC(vmlsl.s32,_a0b,_al2_1,_bl0_1,0)

  152.         VMAC(vmlal.s32,_a0b,_al0_0,_bs0_0,0)

  153.             

  154.         VOP2(vmov,_a1a,_a1b)

  155.         VMAC(vmlal.s32,_a1a,_ah0_1,_bh2_1,1)

  156.         VMAC(vmlal.s32,_a1a,_ah2_0,_bh2_0,1)

  157.         VMAC(vmlal.s32,_a1a,_ah2_1,_bh0_1,1)

  158.         VMAC(vmlal.s32,_a1a,_ah0_0,_bl0_0,1)

  159.             

  160.             VOP2(vswp,_a0b_1,_a0a_0)

  161.             

  162.         VMAC(vmlsl.s32,_a1b,_al0_1,_bl2_1,1)

  163.         VMAC(vmlsl.s32,_a1b,_al2_0,_bl2_0,1)

  164.         VMAC(vmlsl.s32,_a1b,_al2_1,_bl0_1,1)

  165.         VMAC(vmlal.s32,_a1b,_al0_0,_bs0_0,1)

  166.                 

  167.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  168.             VOP3(vsub.i32,_bs0_1,_bl0_1,_bh0_1)

  169.             VOP2(vmovn.i64,_a0b_0,_a0b)

  170.                 

  171.             VOP2(vswp,_a1b_1,_a1a_0)

  172.             VOP3(vadd.i64,_a1b,_a0a,_a1b)

  173.                     

  174.                     

  175.         VMAC(vmull.s32,_a0a,_as2_0,_bs2_1,0)

  176.             VOP2(vmovn.i64,_a0b_1,_a1b)

  177.         VMAC(vmlal.s32,_a0a,_as2_1,_bs2_0,0)

  178.             VOP3(vsra.u64,_a1a,_a1b,"#28")

  179.         VMAC(vmlal.s32,_a0a,_as0_0,_bh0_1,0)

  180.             VOP2(vbic.i32,_a0b,"#0xf0000000")

  181.         VMAC(vmlal.s32,_a0a,_as0_1,_bh0_0,0)

  182.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"

  183.                     

  184.         VMAC(vmull.s32,_a1b,_as2_0,_bs2_1,1)

  185.         VMAC(vmlal.s32,_a1b,_as2_1,_bs2_0,1)

  186.         VMAC(vmlal.s32,_a1b,_as0_0,_bh0_1,1)

  187.         VMAC(vmlal.s32,_a1b,_as0_1,_bh0_0,1)

  188. 

  189.         VOP2(vmov,_a0b_1,_a0a_1)

  190.         VOP3(vadd.i64,_a0b_0,_a0a_0,_a1a_0)

  191.         VOP3(vadd.i64,_a0a_0,_a0a_0,_a1a_1)

  192.         VMAC(vmlal.s32,_a0a,_ah2_0,_bh2_1,0)

  193.         VMAC(vmlal.s32,_a0a,_ah2_1,_bh2_0,0)

  194.         VMAC(vmlal.s32,_a0a,_ah0_0,_bl0_1,0)

  195.         VMAC(vmlal.s32,_a0a,_ah0_1,_bl0_0,0)

  196. 

  197.         VMAC(vmlsl.s32,_a0b,_al2_0,_bl2_1,0)

  198.         VMAC(vmlsl.s32,_a0b,_al2_1,_bl2_0,0)

  199.         VMAC(vmlal.s32,_a0b,_al0_0,_bs0_1,0)

  200.         VMAC(vmlal.s32,_a0b,_al0_1,_bs0_0,0)

  201. 

  202.         VOP2(vmov,_a1a,_a1b)

  203.         VMAC(vmlal.s32,_a1a,_ah2_0,_bh2_1,1)

  204.         VMAC(vmlal.s32,_a1a,_ah2_1,_bh2_0,1)

  205.         VMAC(vmlal.s32,_a1a,_ah0_0,_bl0_1,1)

  206.         VMAC(vmlal.s32,_a1a,_ah0_1,_bl0_0,1)

  207. 

  208.             VOP2(vswp,_a0b_1,_a0a_0)

  209. 

  210.         VMAC(vmlsl.s32,_a1b,_al2_0,_bl2_1,1)

  211.         VMAC(vmlsl.s32,_a1b,_al2_1,_bl2_0,1)

  212.         VMAC(vmlal.s32,_a1b,_al0_0,_bs0_1,1)

  213.         VMAC(vmlal.s32,_a1b,_al0_1,_bs0_0,1)

  214.                                         

  215.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  216.             VOP3(vsub.i32,_bs2_0,_bl2_0,_bh2_0)

  217.             VOP2(vmovn.i64,_a0b_0,_a0b)

  218.                         

  219.             VOP2(vswp,_a1b_1,_a1a_0)

  220.             VOP3(vadd.i64,_a1b,_a0a,_a1b)

  221. 

  222.         VMAC(vmull.s32,_a0a,_as2_1,_bs2_1,0)

  223.             VOP2(vmovn.i64,_a0b_1,_a1b)

  224.         VMAC(vmlal.s32,_a0a,_as0_0,_bh2_0,0)

  225.             VOP3(vsra.u64,_a1a,_a1b,"#28")

  226.         VMAC(vmlal.s32,_a0a,_as0_1,_bh0_1,0)

  227.             VOP2(vbic.i32,_a0b,"#0xf0000000")

  228.         VMAC(vmlal.s32,_a0a,_as2_0,_bh0_0,0)

  229.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"

  230. 

  231.         VMAC(vmull.s32,_a1b,_as2_1,_bs2_1,1)

  232.         VMAC(vmlal.s32,_a1b,_as0_0,_bh2_0,1)

  233.         VMAC(vmlal.s32,_a1b,_as0_1,_bh0_1,1)

  234.         VMAC(vmlal.s32,_a1b,_as2_0,_bh0_0,1)

  235. 

  236.         VOP2(vmov,_a0b_1,_a0a_1)

  237.         VOP3(vadd.i64,_a0b_0,_a0a_0,_a1a_0)

  238.         VOP3(vadd.i64,_a0a_0,_a0a_0,_a1a_1)

  239.         VMAC(vmlal.s32,_a0a,_ah2_1,_bh2_1,0)

  240.         VMAC(vmlal.s32,_a0a,_ah0_0,_bl2_0,0)

  241.         VMAC(vmlal.s32,_a0a,_ah0_1,_bl0_1,0)

  242.         VMAC(vmlal.s32,_a0a,_ah2_0,_bl0_0,0)

  243. 

  244.         VMAC(vmlsl.s32,_a0b,_al2_1,_bl2_1,0)

  245.         VMAC(vmlal.s32,_a0b,_al0_0,_bs2_0,0)

  246.         VMAC(vmlal.s32,_a0b,_al0_1,_bs0_1,0)

  247.         VMAC(vmlal.s32,_a0b,_al2_0,_bs0_0,0)

  248. 

  249.         VOP2(vmov,_a1a,_a1b)

  250.         VMAC(vmlal.s32,_a1a,_ah2_1,_bh2_1,1)

  251.         VMAC(vmlal.s32,_a1a,_ah0_0,_bl2_0,1)

  252.         VMAC(vmlal.s32,_a1a,_ah0_1,_bl0_1,1)

  253.         VMAC(vmlal.s32,_a1a,_ah2_0,_bl0_0,1)

  254. 

  255.             VOP2(vswp,_a0b_1,_a0a_0)

  256. 

  257.         VMAC(vmlsl.s32,_a1b,_al2_1,_bl2_1,1)

  258.         VMAC(vmlal.s32,_a1b,_al0_0,_bs2_0,1)

  259.         VMAC(vmlal.s32,_a1b,_al0_1,_bs0_1,1)

  260.         VMAC(vmlal.s32,_a1b,_al2_0,_bs0_0,1)

  261.                                                                 

  262.             VOP3(vsub.i32,_bs2_1,_bl2_1,_bh2_1)

  263.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  264.             VOP2(vmovn.i64,_a0b_0,_a0b)

  265.                         

  266.             VOP2(vswp,_a1b_1,_a1a_0)

  267.             VOP3(vadd.i64,_a1b,_a0a,_a1b)

  268. 

  269.         VMAC(vmull.s32,_a0a,_as0_0,_bh2_1,0)

  270.             VOP2(vmovn.i64,_a0b_1,_a1b)

  271.         VMAC(vmlal.s32,_a0a,_as0_1,_bh2_0,0)

  272.             VOP3(vsra.u64,_a1a,_a1b,"#28")

  273.         VMAC(vmlal.s32,_a0a,_as2_0,_bh0_1,0)

  274.             VOP2(vbic.i32,_a0b,"#0xf0000000")

  275.         VMAC(vmlal.s32,_a0a,_as2_1,_bh0_0,0)

  276.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"

  277. 

  278.         VMAC(vmull.s32,_a1b,_as0_0,_bh2_1,1)

  279.         VMAC(vmlal.s32,_a1b,_as0_1,_bh2_0,1)

  280.         VMAC(vmlal.s32,_a1b,_as2_0,_bh0_1,1)

  281.         VMAC(vmlal.s32,_a1b,_as2_1,_bh0_0,1)

  282. 

  283.         VOP2(vmov,_a0b_1,_a0a_1)

  284.         VOP3(vadd.i64,_a0b_0,_a0a_0,_a1a_0)

  285.         VOP3(vadd.i64,_a0a_0,_a0a_0,_a1a_1)

  286.         VMAC(vmlal.s32,_a0a,_ah0_0,_bl2_1,0)

  287.         VMAC(vmlal.s32,_a0a,_ah0_1,_bl2_0,0)

  288.         VMAC(vmlal.s32,_a0a,_ah2_0,_bl0_1,0)

  289.         VMAC(vmlal.s32,_a0a,_ah2_1,_bl0_0,0)

  290. 

  291.         VMAC(vmlal.s32,_a0b,_al0_0,_bs2_1,0)

  292.         VMAC(vmlal.s32,_a0b,_al0_1,_bs2_0,0)

  293.         VMAC(vmlal.s32,_a0b,_al2_0,_bs0_1,0)

  294.         VMAC(vmlal.s32,_a0b,_al2_1,_bs0_0,0)

  295. 

  296.         VOP2(vmov,_a1a,_a1b)

  297.         VMAC(vmlal.s32,_a1a,_ah0_0,_bl2_1,1)

  298.         VMAC(vmlal.s32,_a1a,_ah0_1,_bl2_0,1)

  299.         VMAC(vmlal.s32,_a1a,_ah2_0,_bl0_1,1)

  300.         VMAC(vmlal.s32,_a1a,_ah2_1,_bl0_0,1)

  301. 

  302.             VOP2(vswp,_a0b_1,_a0a_0)

  303. 

  304.         VMAC(vmlal.s32,_a1b,_al0_0,_bs2_1,1)

  305.         VMAC(vmlal.s32,_a1b,_al0_1,_bs2_0,1)

  306.         VMAC(vmlal.s32,_a1b,_al2_0,_bs0_1,1)

  307.         VMAC(vmlal.s32,_a1b,_al2_1,_bs0_0,1)

  308.                         

  309.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  310.             VOP2(vmovn.i64,_a0b_0,_a0b)

  311.                                                                                             

  312.             VOP2(vswp,_a1b_1,_a1a_0)

  313.             VOP3(vadd.i64,_a0a,_a0a,_a1b)

  314. 

  315.             VOP2(vmovn.i64,_a0b_1,_a0a)

  316.             VOP3(vsra.u64,_a1a,_a0a,"#28")

  317.                                                                                             

  318.             VOP2(vbic.i32,_a0b,"#0xf0000000") 

  319.                                                                                             

  320.         VOP2(vswp,_a1a_0,_a1a_1)

  321.                                                                                             

  322.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"  

  323.             "sub %[c], #64" "\n\t"

  324.                                                                                                 

  325.         VOP3(vadd.i64,_a1a_1,_a1a_1,_a1a_0)

  326.         

  327.             "vldmia %[c], {"_a0a_0", "_a0a_1", "_a0b_0"}" "\n\t"

  328.             VOP2(vaddw.s32,_a1a,_a0a_0)

  329.             VOP2(vmovn.i64,_a0a_0,_a1a)

  330.             VOP2(vshr.s64,_a1a,"#28")

  331.                                                 

  332.             VOP2(vaddw.s32,_a1a,_a0a_1)

  333.             VOP2(vmovn.i64,_a0a_1,_a1a)

  334.             VOP2(vshr.s64,_a1a,"#28")

  335.                                                                                                     

  336.             VOP2(vbic.i32,_a0a,"#0xf0000000")

  337.                                                 

  338.             VOP2(vaddw.s32,_a1a,_a0b_0) 

  339.             VOP2(vmovn.i64,_a0b_0,_a1a)

  340.             

  341.             "vstmia %[c], {"_a0a_0", "_a0a_1", "_a0b_0"}" "\n\t"

  342.         

  343.         : [a]"+r"(as)

  344.         , [b]"+r"(bs)

  345.         , [c]"+r"(vc)

  346.                             

  347.         :: "q0","q1","q2","q3",

  348.             "q4","q5","q6","q7",

  349.             "q8","q9","q10","q11",

  350.             "q12","q13","q14","q15",

  351.             "memory"

  352.     );

  353. }

  354. 

  355. void gf_sqr (gf_s *__restrict__ cs, const gf bs) {

  356.     int32x2_t *vc = (int32x2_t*) cs->limb;

  357. 

  358.     __asm__ __volatile__ (

  359.         "vld2.32 {"_bl0_0","_bl0_1","_bh0_0","_bh0_1"}, [%[b],:128]!" "\n\t"

  360.         VOP3(vadd.i32,_bs0_1,_bl0_1,_bh0_1) /* 0 .. 2^30 */

  361.         VOP3(vsub.i32,_bs0_0,_bl0_0,_bh0_0) /* +- 2^29 */

  362.         VOP3(vadd.i32,_as0,_bl0,_bh0)       /* 0 .. 2^30 */

  363.             

  364.         "vld2.32 {"_bl2_0","_bl2_1","_bh2_0","_bh2_1"}, [%[b],:128]!" "\n\t"

  365.         VOP3(vadd.i32,_bs2,_bl2,_bh2)       /* 0 .. 2^30 */

  366.         VOP2(vmov,_as2,_bs2)

  367.         

  368.         VMAC(vqdmull.s32,_a0b,_as0_1,_bs2_1,0) /* 0 .. 8 * 2^58.  danger for vqdmlal is 32 */

  369.         VMAC(vmlal.s32,_a0b,_as2_0,_bs2_0,0)   /* 0 .. 12 */

  370.         VMAC(vmlal.s32,_a0b,_as0_0,_bh0_0,0)   /* 0 .. 14 */

  371.             

  372.         VMAC(vqdmull.s32,_a1b,_as0_1,_bs2_1,1) /* 0 .. 8 */

  373.         VMAC(vmlal.s32,_a1b,_as2_0,_bs2_0,1)   /* 0 .. 14 */

  374.         VMAC(vmlal.s32,_a1b,_as0_0,_bh0_0,1)   /* 0 .. 16 */

  375.             

  376.         VOP2(vmov,_a0a,_a0b)                   /* 0 .. 14 */

  377.         VMAC(vqdmlal.s32,_a0a,_bh0_1,_bh2_1,0) /* 0 .. 16 */

  378.         VMAC(vmlal.s32,_a0a,_bh2_0,_bh2_0,0)   /* 0 .. 17 */

  379.         VMAC(vmlal.s32,_a0a,_bh0_0,_bl0_0,0)   /* 0 .. 18 */

  380.             

  381.         VMAC(vqdmlsl.s32,_a0b,_bl0_1,_bl2_1,0) /*-2 .. 14 */

  382.         VMAC(vmlsl.s32,_a0b,_bl2_0,_bl2_0,0)   /*-3 .. 14 */

  383.         VMAC(vmlal.s32,_a0b,_bl0_0,_bs0_0,0)   /*-4 .. 15 */

  384.             

  385.         VOP2(vmov,_a1a,_a1b)

  386.         VMAC(vqdmlal.s32,_a1a,_bh0_1,_bh2_1,1) /* 0 .. 18 */

  387.         VMAC(vmlal.s32,_a1a,_bh2_0,_bh2_0,1)   /* 0 .. 19 */

  388.         VMAC(vmlal.s32,_a1a,_bh0_0,_bl0_0,1)   /* 0 .. 20 */

  389.             

  390.             VOP2(vswp,_a0b_1,_a0a_0)

  391.             

  392.         VMAC(vqdmlsl.s32,_a1b,_bl0_1,_bl2_1,1) /*-2 .. 16 */

  393.         VMAC(vmlsl.s32,_a1b,_bl2_0,_bl2_0,1)   /*-3 .. 16 */

  394.         VMAC(vmlal.s32,_a1b,_bl0_0,_bs0_0,1)   /*-4 .. 17 */

  395.                 

  396.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  397.             VOP3(vsub.i32,_bs0_1,_bl0_1,_bh0_1)

  398.             VOP2(vmovn.i64,_a0b_0,_a0b)

  399.                 

  400.             VOP2(vswp,_a1b_1,_a1a_0)

  401.             VOP3(vadd.i64,_a1b,_a0a,_a1b)

  402.                     

  403.                     

  404.         VMAC(vqdmull.s32,_a0a,_as2_0,_bs2_1,0) /* 0 .. 8 */

  405.             VOP2(vmovn.i64,_a0b_1,_a1b)

  406.             VOP3(vsra.u64,_a1a,_a1b,"#28")

  407.         VMAC(vqdmlal.s32,_a0a,_as0_0,_bh0_1,0) /* 0 .. 12 */

  408.             VOP2(vbic.i32,_a0b,"#0xf0000000")

  409.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"

  410.                     

  411.         VMAC(vqdmull.s32,_a1b,_as2_0,_bs2_1,1) /* 0 .. 8 */

  412.         VMAC(vqdmlal.s32,_a1b,_as0_0,_bh0_1,1) /* 0 .. 12 */

  413. 

  414.         VOP2(vmov,_a0b,_a0a)               /* 0 .. 12 */

  415.         VMAC(vqdmlal.s32,_a0a,_bh2_0,_bh2_1,0) /* 0 .. 14 */

  416.         VMAC(vqdmlal.s32,_a0a,_bh0_0,_bl0_1,0) /* 0 .. 16 */

  417. 

  418.         VMAC(vqdmlsl.s32,_a0b,_bl2_0,_bl2_1,0) /*-2 .. 12 */

  419.         VMAC(vqdmlal.s32,_a0b,_bl0_0,_bs0_1,0) /*-4 .. 14 */

  420.         VOP3(vadd.i64,_a0a_0,_a0a_0,_a1a_1)

  421.         VOP3(vadd.i64,_a0b_0,_a0b_0,_a1a_0)

  422. 

  423.         VOP2(vmov,_a1a,_a1b)                   /* 0 .. 12 */

  424.         VMAC(vqdmlal.s32,_a1a,_bh2_0,_bh2_1,1) /* 0 .. 14 */

  425.         VMAC(vqdmlal.s32,_a1a,_bh0_0,_bl0_1,1) /* 0 .. 16 */

  426. 

  427.             VOP2(vswp,_a0b_1,_a0a_0)

  428. 

  429.         VMAC(vqdmlsl.s32,_a1b,_bl2_0,_bl2_1,1) /*-2 .. 12 */

  430.         VMAC(vqdmlal.s32,_a1b,_bl0_0,_bs0_1,1) /*-4 .. 14 */

  431.                                         

  432.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  433.             VOP3(vsub.i32,_bs2_0,_bl2_0,_bh2_0)

  434.             VOP2(vmovn.i64,_a0b_0,_a0b)

  435.                         

  436.             VOP2(vswp,_a1b_1,_a1a_0)

  437.             VOP3(vadd.i64,_a1b,_a0a,_a1b)

  438. 

  439.         VMAC(vmull.s32,_a0a,_as2_1,_bs2_1,0)

  440.             VOP2(vmovn.i64,_a0b_1,_a1b)

  441.         VMAC(vqdmlal.s32,_a0a,_as0_0,_bh2_0,0)

  442.             VOP3(vsra.u64,_a1a,_a1b,"#28")

  443.         VMAC(vmlal.s32,_a0a,_as0_1,_bh0_1,0)

  444.             VOP2(vbic.i32,_a0b,"#0xf0000000")

  445.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"

  446. 

  447.         VMAC(vmull.s32,_a1b,_as2_1,_bs2_1,1)

  448.         VMAC(vqdmlal.s32,_a1b,_as0_0,_bh2_0,1)

  449.         VMAC(vmlal.s32,_a1b,_as0_1,_bh0_1,1)

  450. 

  451.         VOP2(vmov,_a0b_1,_a0a_1)

  452.         VOP3(vadd.i64,_a0b_0,_a0a_0,_a1a_0)

  453.         VOP3(vadd.i64,_a0a_0,_a0a_0,_a1a_1)

  454.         VMAC(vmlal.s32,_a0a,_bh2_1,_bh2_1,0)

  455.         VMAC(vqdmlal.s32,_a0a,_bh0_0,_bl2_0,0)

  456.         VMAC(vmlal.s32,_a0a,_bh0_1,_bl0_1,0)

  457. 

  458.         VMAC(vmlsl.s32,_a0b,_bl2_1,_bl2_1,0)

  459.         VMAC(vqdmlal.s32,_a0b,_bl0_0,_bs2_0,0)

  460.         VMAC(vmlal.s32,_a0b,_bl0_1,_bs0_1,0)

  461. 

  462.         VOP2(vmov,_a1a,_a1b)

  463.         VMAC(vmlal.s32,_a1a,_bh2_1,_bh2_1,1)

  464.         VMAC(vqdmlal.s32,_a1a,_bh0_0,_bl2_0,1)

  465.         VMAC(vmlal.s32,_a1a,_bh0_1,_bl0_1,1)

  466. 

  467.             VOP2(vswp,_a0b_1,_a0a_0)

  468. 

  469.         VMAC(vmlsl.s32,_a1b,_bl2_1,_bl2_1,1)

  470.         VMAC(vqdmlal.s32,_a1b,_bl0_0,_bs2_0,1)

  471.         VMAC(vmlal.s32,_a1b,_bl0_1,_bs0_1,1)

  472.                                                                 

  473.             VOP3(vsub.i32,_bs2_1,_bl2_1,_bh2_1)

  474.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  475.             VOP2(vmovn.i64,_a0b_0,_a0b)

  476.                         

  477.             VOP2(vswp,_a1b_1,_a1a_0)

  478.             VOP3(vadd.i64,_a1b,_a0a,_a1b)

  479. 

  480.         VMAC(vqdmull.s32,_a0a,_as0_0,_bh2_1,0)

  481.             VOP2(vmovn.i64,_a0b_1,_a1b)

  482.             VOP3(vsra.u64,_a1a,_a1b,"#28")

  483.         VMAC(vqdmlal.s32,_a0a,_as2_0,_bh0_1,0)

  484.             VOP2(vbic.i32,_a0b,"#0xf0000000")

  485.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"

  486. 

  487.         VMAC(vqdmull.s32,_a1b,_as0_0,_bh2_1,1)

  488.         VMAC(vqdmlal.s32,_a1b,_as2_0,_bh0_1,1)

  489. 

  490.         VOP2(vmov,_a0b_1,_a0a_1)

  491.         VOP3(vadd.i64,_a0b_0,_a0a_0,_a1a_0)

  492.         VOP3(vadd.i64,_a0a_0,_a0a_0,_a1a_1)

  493.         VMAC(vqdmlal.s32,_a0a,_bh0_0,_bl2_1,0)

  494.         VMAC(vqdmlal.s32,_a0a,_bh2_0,_bl0_1,0)

  495. 

  496.         VMAC(vqdmlal.s32,_a0b,_bl0_0,_bs2_1,0)

  497.         VMAC(vqdmlal.s32,_a0b,_bl2_0,_bs0_1,0)

  498. 

  499.         VOP2(vmov,_a1a,_a1b)

  500.         VMAC(vqdmlal.s32,_a1a,_bh0_0,_bl2_1,1)

  501.         VMAC(vqdmlal.s32,_a1a,_bh2_0,_bl0_1,1)

  502. 

  503.             VOP2(vswp,_a0b_1,_a0a_0)

  504. 

  505.         VMAC(vqdmlal.s32,_a1b,_bl0_0,_bs2_1,1)

  506.         VMAC(vqdmlal.s32,_a1b,_bl2_0,_bs0_1,1)

  507.                         

  508.             VOP3(vsra.u64,_a0a,_a0b,"#28")

  509.             VOP2(vmovn.i64,_a0b_0,_a0b)

  510.                                                                                             

  511.             VOP2(vswp,_a1b_1,_a1a_0)

  512.             VOP3(vadd.i64,_a0a,_a0a,_a1b)

  513. 

  514.             VOP2(vmovn.i64,_a0b_1,_a0a)

  515.             VOP3(vsra.u64,_a1a,_a0a,"#28")

  516.                                                                                             

  517.             VOP2(vbic.i32,_a0b,"#0xf0000000") 

  518.                                                                                             

  519.         VOP2(vswp,_a1a_0,_a1a_1)

  520.                                                                                             

  521.             "vstmia %[c]!, {"_a0b_0", "_a0b_1"}" "\n\t"  

  522.             "sub %[c], #64" "\n\t"

  523.                                                                                                 

  524.         VOP3(vadd.i64,_a1a_1,_a1a_1,_a1a_0)

  525.         

  526.             "vldmia %[c], {"_a0a_0", "_a0a_1", "_a0b_0"}" "\n\t"

  527.             VOP2(vaddw.s32,_a1a,_a0a_0)

  528.             VOP2(vmovn.i64,_a0a_0,_a1a)

  529.             VOP2(vshr.s64,_a1a,"#28")

  530.                                                 

  531.             VOP2(vaddw.s32,_a1a,_a0a_1)

  532.             VOP2(vmovn.i64,_a0a_1,_a1a)

  533.             VOP2(vshr.s64,_a1a,"#28")

  534.                                                                                                     

  535.             VOP2(vbic.i32,_a0a,"#0xf0000000")

  536.                                                 

  537.             VOP2(vaddw.s32,_a1a,_a0b_0) 

  538.             VOP2(vmovn.i64,_a0b_0,_a1a)

  539.             

  540.             "vstmia %[c], {"_a0a_0", "_a0a_1", "_a0b_0"}" "\n\t"

  541.         

  542.         : [b]"+r"(bs)

  543.         , [c]"+r"(vc)

  544.                             

  545.         :: "q0","q1","q2","q3",

  546.             "q4","q5","q6","q7",

  547.             "q12","q13","q14","q15",

  548.             "memory"

  549.     );

  550. }

  551. 

  552. void gf_mulw_unsigned (gf_s *__restrict__ cs, const gf as, uint32_t b) { 

  553.     uint32x2_t vmask = {(1<<28) - 1, (1<<28)-1};

  554.     assert(b<(1<<28));

  555.     

  556.     uint64x2_t accum;

  557.     const uint32x2_t *va = (const uint32x2_t *) as->limb;

  558.     uint32x2_t *vo = (uint32x2_t *) cs->limb;

  559.     uint32x2_t vc, vn;

  560.     uint32x2_t vb = {b, 0};

  561.     

  562.     vc = va[0];

  563.     accum = vmull_lane_u32(accum, vc, vb, 0);

  564.     vo[0] = vmovn_u64(accum) & vmask;

  565.     accum = vshrq_n_u64(accum,28);

  566.     

  567.     /* PERF: the right way to do this is to reduce behind, i.e.

  568.      * vmull + vmlal round 0

  569.      * vmull + vmlal round 1

  570.      * vmull + vmlal round 2

  571.      * vsraq round 0, 1

  572.      * vmull + vmlal round 3

  573.      * vsraq round 1, 2

  574.      * ...

  575.      */

  576.     

  577.     int i;

  578.     for (i=1; i<8; i++) {

  579.         vn = va[i];

  580.         accum = vmlal_lane_u32(accum, vn, vb, 0);

  581.         vo[i] = vmovn_u64(accum) & vmask;

  582.         accum = vshrq_n_u64(accum,28);

  583.         vc = vn;

  584.     }

  585.         

  586.     accum = xx_vaddup_u64(vrev128_u64(accum));

  587.     accum = vaddw_u32(accum, vo[0]);

  588.     vo[0] = vmovn_u64(accum) & vmask;

  589.     

  590.     accum = vshrq_n_u64(accum,28);

  591.     vo[1] += vmovn_u64(accum);

  592. }