// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT-0

// ----------------------------------------------------------------------------
// Montgomery multiply, z := (x * y / 2^384) mod p_384
// Inputs x[6], y[6]; output z[6]
//
//    extern void bignum_montmul_p384_alt
//     (uint64_t z[static 6], uint64_t x[static 6], uint64_t y[static 6]);
//
// Does z := (2^{-384} * x * y) mod p_384, assuming that the inputs x and y
// satisfy x * y <= 2^384 * p_384 (in particular this is true if we are in
// the "usual" case x < p_384 and y < p_384).
//
// Standard ARM ABI: X0 = z, X1 = x, X2 = y
// ----------------------------------------------------------------------------
#include "_internal_s2n_bignum.h"

        S2N_BN_SYM_VISIBILITY_DIRECTIVE(bignum_montmul_p384_alt)
        S2N_BN_SYM_PRIVACY_DIRECTIVE(bignum_montmul_p384_alt)
        .text
        .balign 4

// ---------------------------------------------------------------------------
// Core one-step "short" Montgomery reduction macro. Takes input in
// [d5;d4;d3;d2;d1;d0] and returns result in [d6;d5;d4;d3;d2;d1],
// adding to the existing contents of [d5;d4;d3;d2;d1]. It is fine
// for d6 to be the same register as d0.
//
// We want to add (2^384 - 2^128 - 2^96 + 2^32 - 1) * w
// where w = [d0 + (d0<<32)] mod 2^64
// ---------------------------------------------------------------------------

#define montreds(d6,d5,d4,d3,d2,d1,d0, t3,t2,t1)                            \
/* Our correction multiplier is w = [d0 + (d0<<32)] mod 2^64            */  \
/* Store it in d6 to make the 2^384 * w contribution already            */  \
        lsl     t1, d0, #32;                                        \
        add     d6, t1, d0;                                         \
/* Now let [t3;t2;t1;-] = (2^384 - p_384) * w                    */         \
/* We know the lowest word will cancel d0 so we don't need it    */         \
        mov     t1, #0xffffffff00000001;                            \
        umulh   t1, t1, d6;                                         \
        mov     t2, #0x00000000ffffffff;                            \
        mul     t3, t2, d6;                                         \
        umulh   t2, t2, d6;                                         \
        adds    t1, t1, t3;                                         \
        adcs    t2, t2, d6;                                         \
        adc     t3, xzr, xzr;                                       \
/* Now add it, by subtracting from 2^384 * w + x */                         \
        subs    d1, d1, t1;                                         \
        sbcs    d2, d2, t2;                                         \
        sbcs    d3, d3, t3;                                         \
        sbcs    d4, d4, xzr;                                        \
        sbcs    d5, d5, xzr;                                        \
        sbc     d6, d6, xzr


#define z x0
#define x x1
#define y x2

// These are repeated mod 2 as we load pairs of inputs

#define a0 x3
#define a1 x4
#define a2 x3
#define a3 x4
#define a4 x3
#define a5 x4

#define b0 x5
#define b1 x6
#define b2 x7
#define b3 x8
#define b4 x9
#define b5 x10

#define l x11

#define u0 x12
#define u1 x13
#define u2 x14
#define u3 x15
#define u4 x16
#define u5 x17
#define u6 x19
#define u7 x20
#define u8 x21
#define u9 x22
#define u10 x2 // same as y
#define u11 x1 // same as x
#define h b5 // same as b5

S2N_BN_SYMBOL(bignum_montmul_p384_alt):

// Save more registers

        stp     x19, x20, [sp, #-16]!
        stp     x21, x22, [sp, #-16]!

// Load operands and set up row 0 = [u6;...;u0] = a0 * [b5;...;b0]

        ldp     a0, a1, [x]
        ldp     b0, b1, [y]

        mul     u0, a0, b0
        umulh   u1, a0, b0
        mul     l, a0, b1
        umulh   u2, a0, b1
        adds    u1, u1, l

        ldp     b2, b3, [y, #16]

        mul     l, a0, b2
        umulh   u3, a0, b2
        adcs    u2, u2, l

        mul     l, a0, b3
        umulh   u4, a0, b3
        adcs    u3, u3, l

        ldp     b4, b5, [y, #32]

        mul     l, a0, b4
        umulh   u5, a0, b4
        adcs    u4, u4, l

        mul     l, a0, b5
        umulh   u6, a0, b5
        adcs    u5, u5, l

        adc     u6, u6, xzr

// Row 1 = [u7;...;u0] = [a1;a0] * [b5;...;b0]

        mul     l, a1, b0
        adds    u1, u1, l
        mul     l, a1, b1
        adcs    u2, u2, l
        mul     l, a1, b2
        adcs    u3, u3, l
        mul     l, a1, b3
        adcs    u4, u4, l
        mul     l, a1, b4
        adcs    u5, u5, l
        mul     l, a1, b5
        adcs    u6, u6, l
        cset    u7, cs

        umulh   l, a1, b0
        adds    u2, u2, l
        umulh   l, a1, b1
        adcs    u3, u3, l
        umulh   l, a1, b2
        adcs    u4, u4, l
        umulh   l, a1, b3
        adcs    u5, u5, l
        umulh   l, a1, b4
        adcs    u6, u6, l
        umulh   l, a1, b5
        adc     u7, u7, l

// Row 2 = [u8;...;u0] = [a2;a1;a0] * [b5;...;b0]

        ldp     a2, a3, [x, #16]

        mul     l, a2, b0
        adds    u2, u2, l
        mul     l, a2, b1
        adcs    u3, u3, l
        mul     l, a2, b2
        adcs    u4, u4, l
        mul     l, a2, b3
        adcs    u5, u5, l
        mul     l, a2, b4
        adcs    u6, u6, l
        mul     l, a2, b5
        adcs    u7, u7, l
        cset    u8, cs

        umulh   l, a2, b0
        adds    u3, u3, l
        umulh   l, a2, b1
        adcs    u4, u4, l
        umulh   l, a2, b2
        adcs    u5, u5, l
        umulh   l, a2, b3
        adcs    u6, u6, l
        umulh   l, a2, b4
        adcs    u7, u7, l
        umulh   l, a2, b5
        adc     u8, u8, l

// Row 3 = [u9;...;u0] = [a3;a2;a1;a0] * [b5;...;b0]

        mul     l, a3, b0
        adds    u3, u3, l
        mul     l, a3, b1
        adcs    u4, u4, l
        mul     l, a3, b2
        adcs    u5, u5, l
        mul     l, a3, b3
        adcs    u6, u6, l
        mul     l, a3, b4
        adcs    u7, u7, l
        mul     l, a3, b5
        adcs    u8, u8, l
        cset    u9, cs

        umulh   l, a3, b0
        adds    u4, u4, l
        umulh   l, a3, b1
        adcs    u5, u5, l
        umulh   l, a3, b2
        adcs    u6, u6, l
        umulh   l, a3, b3
        adcs    u7, u7, l
        umulh   l, a3, b4
        adcs    u8, u8, l
        umulh   l, a3, b5
        adc     u9, u9, l

// Row 4 = [u10;...;u0] = [a4;a3;a2;a1;a0] * [b5;...;b0]

        ldp     a4, a5, [x, #32]

        mul     l, a4, b0
        adds    u4, u4, l
        mul     l, a4, b1
        adcs    u5, u5, l
        mul     l, a4, b2
        adcs    u6, u6, l
        mul     l, a4, b3
        adcs    u7, u7, l
        mul     l, a4, b4
        adcs    u8, u8, l
        mul     l, a4, b5
        adcs    u9, u9, l
        cset    u10, cs

        umulh   l, a4, b0
        adds    u5, u5, l
        umulh   l, a4, b1
        adcs    u6, u6, l
        umulh   l, a4, b2
        adcs    u7, u7, l
        umulh   l, a4, b3
        adcs    u8, u8, l
        umulh   l, a4, b4
        adcs    u9, u9, l
        umulh   l, a4, b5
        adc     u10, u10, l

// Row 5 = [u11;...;u0] = [a5;a4;a3;a2;a1;a0] * [b5;...;b0]

        mul     l, a5, b0
        adds    u5, u5, l
        mul     l, a5, b1
        adcs    u6, u6, l
        mul     l, a5, b2
        adcs    u7, u7, l
        mul     l, a5, b3
        adcs    u8, u8, l
        mul     l, a5, b4
        adcs    u9, u9, l
        mul     l, a5, b5
        adcs    u10, u10, l
        cset    u11, cs

        umulh   l, a5, b0
        adds    u6, u6, l
        umulh   l, a5, b1
        adcs    u7, u7, l
        umulh   l, a5, b2
        adcs    u8, u8, l
        umulh   l, a5, b3
        adcs    u9, u9, l
        umulh   l, a5, b4
        adcs    u10, u10, l
        umulh   l, a5, b5
        adc     u11, u11, l

// Montgomery rotate the low half

        montreds(u0,u5,u4,u3,u2,u1,u0, b0,b1,b2)
        montreds(u1,u0,u5,u4,u3,u2,u1, b0,b1,b2)
        montreds(u2,u1,u0,u5,u4,u3,u2, b0,b1,b2)
        montreds(u3,u2,u1,u0,u5,u4,u3, b0,b1,b2)
        montreds(u4,u3,u2,u1,u0,u5,u4, b0,b1,b2)
        montreds(u5,u4,u3,u2,u1,u0,u5, b0,b1,b2)

// Add up the high and low parts as [h; u5;u4;u3;u2;u1;u0] = z

        adds    u0, u0, u6
        adcs    u1, u1, u7
        adcs    u2, u2, u8
        adcs    u3, u3, u9
        adcs    u4, u4, u10
        adcs    u5, u5, u11
        adc     h, xzr, xzr

// Now add [h; u11;u10;u9;u8;u7;u6] = z + (2^384 - p_384)

        mov     l, #0xffffffff00000001
        adds    u6, u0, l
        mov     l, #0x00000000ffffffff
        adcs    u7, u1, l
        mov     l, #0x0000000000000001
        adcs    u8, u2, l
        adcs    u9, u3, xzr
        adcs    u10, u4, xzr
        adcs    u11, u5, xzr
        adcs    h, h, xzr

// Now z >= p_384 iff h is nonzero, so select accordingly

        csel    u0, u0, u6, eq
        csel    u1, u1, u7, eq
        csel    u2, u2, u8, eq
        csel    u3, u3, u9, eq
        csel    u4, u4, u10, eq
        csel    u5, u5, u11, eq

// Store back final result

        stp     u0, u1, [z]
        stp     u2, u3, [z, #16]
        stp     u4, u5, [z, #32]

// Restore registers

        ldp     x21, x22, [sp], #16
        ldp     x19, x20, [sp], #16

        ret

#if defined(__linux__) && defined(__ELF__)
.section .note.GNU-stack,"",%progbits
#endif
